2024-11-16 12:50:57,065 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 2024-11-16 12:50:57,085 main DEBUG Took 0.017282 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-16 12:50:57,085 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-16 12:50:57,086 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-16 12:50:57,087 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-16 12:50:57,089 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 12:50:57,098 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-16 12:50:57,114 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 12:50:57,115 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 12:50:57,116 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 12:50:57,116 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 12:50:57,117 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 12:50:57,117 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 12:50:57,118 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 12:50:57,118 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 12:50:57,119 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 12:50:57,119 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 12:50:57,120 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 12:50:57,120 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 12:50:57,121 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 12:50:57,121 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 12:50:57,121 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 12:50:57,122 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 12:50:57,122 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 12:50:57,122 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 12:50:57,123 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 12:50:57,123 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 12:50:57,124 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 12:50:57,124 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 12:50:57,124 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 12:50:57,125 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 12:50:57,125 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 12:50:57,126 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-16 12:50:57,128 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 12:50:57,130 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-16 12:50:57,132 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-16 12:50:57,133 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-16 12:50:57,135 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-16 12:50:57,135 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-16 12:50:57,146 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-16 12:50:57,150 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-16 12:50:57,152 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-16 12:50:57,152 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-16 12:50:57,153 main DEBUG createAppenders(={Console}) 2024-11-16 12:50:57,154 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 initialized 2024-11-16 12:50:57,155 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 2024-11-16 12:50:57,155 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 OK. 2024-11-16 12:50:57,156 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-16 12:50:57,156 main DEBUG OutputStream closed 2024-11-16 12:50:57,156 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-16 12:50:57,157 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-16 12:50:57,157 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@57cf54e1 OK 2024-11-16 12:50:57,253 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-16 12:50:57,256 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-16 12:50:57,258 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-16 12:50:57,259 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-16 12:50:57,260 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-16 12:50:57,261 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-16 12:50:57,261 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-16 12:50:57,262 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-16 12:50:57,262 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-16 12:50:57,263 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-16 12:50:57,263 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-16 12:50:57,264 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-16 12:50:57,264 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-16 12:50:57,264 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-16 12:50:57,265 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-16 12:50:57,265 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-16 12:50:57,266 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-16 12:50:57,267 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-16 12:50:57,270 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16 12:50:57,270 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6f63b475) with optional ClassLoader: null 2024-11-16 12:50:57,271 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-16 12:50:57,272 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6f63b475] started OK. 2024-11-16T12:50:57,288 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestZooKeeper timeout: 13 mins 2024-11-16 12:50:57,292 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-16 12:50:57,292 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16T12:50:57,611 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2 2024-11-16T12:50:57,631 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/cluster_35bff147-7ae7-a41a-0bf2-848ec5006277, deleteOnExit=true 2024-11-16T12:50:57,633 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/test.cache.data in system properties and HBase conf 2024-11-16T12:50:57,634 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/hadoop.tmp.dir in system properties and HBase conf 2024-11-16T12:50:57,635 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/hadoop.log.dir in system properties and HBase conf 2024-11-16T12:50:57,636 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-16T12:50:57,637 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-16T12:50:57,637 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-16T12:50:57,757 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-16T12:50:57,854 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T12:50:57,858 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-16T12:50:57,859 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-16T12:50:57,859 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-16T12:50:57,860 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T12:50:57,860 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-16T12:50:57,861 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-16T12:50:57,861 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T12:50:57,861 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T12:50:57,862 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-16T12:50:57,862 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/nfs.dump.dir in system properties and HBase conf 2024-11-16T12:50:57,863 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/java.io.tmpdir in system properties and HBase conf 2024-11-16T12:50:57,863 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T12:50:57,864 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-16T12:50:57,864 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-16T12:50:59,190 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-16T12:50:59,272 INFO [Time-limited test {}] log.Log(170): Logging initialized @2929ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-16T12:50:59,355 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T12:50:59,439 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T12:50:59,482 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T12:50:59,483 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T12:50:59,485 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-16T12:50:59,500 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T12:50:59,504 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@381d428e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/hadoop.log.dir/,AVAILABLE} 2024-11-16T12:50:59,505 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2fb34cd2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T12:50:59,705 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2984ac49{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/java.io.tmpdir/jetty-localhost-33845-hadoop-hdfs-3_4_1-tests_jar-_-any-7554326932291035442/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T12:50:59,713 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@53cf5163{HTTP/1.1, (http/1.1)}{localhost:33845} 2024-11-16T12:50:59,713 INFO [Time-limited test {}] server.Server(415): Started @3371ms 2024-11-16T12:51:00,238 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T12:51:00,247 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T12:51:00,248 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T12:51:00,248 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T12:51:00,249 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-16T12:51:00,250 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@52505b99{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/hadoop.log.dir/,AVAILABLE} 2024-11-16T12:51:00,250 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6dfa801a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T12:51:00,352 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7bd6cc9c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/java.io.tmpdir/jetty-localhost-33675-hadoop-hdfs-3_4_1-tests_jar-_-any-818078408049685097/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T12:51:00,352 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@66909f07{HTTP/1.1, (http/1.1)}{localhost:33675} 2024-11-16T12:51:00,353 INFO [Time-limited test {}] server.Server(415): Started @4011ms 2024-11-16T12:51:00,404 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-16T12:51:00,531 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T12:51:00,539 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T12:51:00,550 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T12:51:00,550 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T12:51:00,551 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-16T12:51:00,554 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@9b56b65{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/hadoop.log.dir/,AVAILABLE} 2024-11-16T12:51:00,556 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3d32cb4f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T12:51:00,691 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@68f66b07{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/java.io.tmpdir/jetty-localhost-40083-hadoop-hdfs-3_4_1-tests_jar-_-any-12346077947304925052/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T12:51:00,692 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6435c1b7{HTTP/1.1, (http/1.1)}{localhost:40083} 2024-11-16T12:51:00,693 INFO [Time-limited test {}] server.Server(415): Started @4351ms 2024-11-16T12:51:00,695 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-16T12:51:01,515 WARN [Thread-99 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/cluster_35bff147-7ae7-a41a-0bf2-848ec5006277/data/data4/current/BP-1388834573-172.17.0.2-1731761458475/current, will proceed with Du for space computation calculation, 2024-11-16T12:51:01,515 WARN [Thread-101 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/cluster_35bff147-7ae7-a41a-0bf2-848ec5006277/data/data2/current/BP-1388834573-172.17.0.2-1731761458475/current, will proceed with Du for space computation calculation, 2024-11-16T12:51:01,515 WARN [Thread-100 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/cluster_35bff147-7ae7-a41a-0bf2-848ec5006277/data/data1/current/BP-1388834573-172.17.0.2-1731761458475/current, will proceed with Du for space computation calculation, 2024-11-16T12:51:01,515 WARN [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/cluster_35bff147-7ae7-a41a-0bf2-848ec5006277/data/data3/current/BP-1388834573-172.17.0.2-1731761458475/current, will proceed with Du for space computation calculation, 2024-11-16T12:51:01,545 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-16T12:51:01,546 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-16T12:51:01,590 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3cf0ab080f443cf9 with lease ID 0xc236dc4edcb346d: Processing first storage report for DS-bb03cd21-30d0-48b1-be65-bcbb42111fab from datanode DatanodeRegistration(127.0.0.1:41745, datanodeUuid=e66f21fb-f632-45fa-bc72-6cafd26ecc03, infoPort=35745, infoSecurePort=0, ipcPort=42061, storageInfo=lv=-57;cid=testClusterID;nsid=1174441748;c=1731761458475) 2024-11-16T12:51:01,591 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3cf0ab080f443cf9 with lease ID 0xc236dc4edcb346d: from storage DS-bb03cd21-30d0-48b1-be65-bcbb42111fab node DatanodeRegistration(127.0.0.1:41745, datanodeUuid=e66f21fb-f632-45fa-bc72-6cafd26ecc03, infoPort=35745, infoSecurePort=0, ipcPort=42061, storageInfo=lv=-57;cid=testClusterID;nsid=1174441748;c=1731761458475), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-16T12:51:01,592 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x50ba6d65eac51891 with lease ID 0xc236dc4edcb346c: Processing first storage report for DS-7d986ccd-af39-4885-87c9-4c16bd6f98ad from datanode DatanodeRegistration(127.0.0.1:40349, datanodeUuid=ba7bc4ea-b82c-4d51-b7d8-2e7d719b62fa, infoPort=46783, infoSecurePort=0, ipcPort=45535, storageInfo=lv=-57;cid=testClusterID;nsid=1174441748;c=1731761458475) 2024-11-16T12:51:01,592 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x50ba6d65eac51891 with lease ID 0xc236dc4edcb346c: from storage DS-7d986ccd-af39-4885-87c9-4c16bd6f98ad node DatanodeRegistration(127.0.0.1:40349, datanodeUuid=ba7bc4ea-b82c-4d51-b7d8-2e7d719b62fa, infoPort=46783, infoSecurePort=0, ipcPort=45535, storageInfo=lv=-57;cid=testClusterID;nsid=1174441748;c=1731761458475), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T12:51:01,592 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3cf0ab080f443cf9 with lease ID 0xc236dc4edcb346d: Processing first storage report for DS-98a645c3-03c4-4c94-846d-c67832cdd154 from datanode DatanodeRegistration(127.0.0.1:41745, datanodeUuid=e66f21fb-f632-45fa-bc72-6cafd26ecc03, infoPort=35745, infoSecurePort=0, ipcPort=42061, storageInfo=lv=-57;cid=testClusterID;nsid=1174441748;c=1731761458475) 2024-11-16T12:51:01,592 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3cf0ab080f443cf9 with lease ID 0xc236dc4edcb346d: from storage DS-98a645c3-03c4-4c94-846d-c67832cdd154 node DatanodeRegistration(127.0.0.1:41745, datanodeUuid=e66f21fb-f632-45fa-bc72-6cafd26ecc03, infoPort=35745, infoSecurePort=0, ipcPort=42061, storageInfo=lv=-57;cid=testClusterID;nsid=1174441748;c=1731761458475), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-16T12:51:01,593 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x50ba6d65eac51891 with lease ID 0xc236dc4edcb346c: Processing first storage report for DS-90e24ac2-369e-4600-add3-9f2117b6171a from datanode DatanodeRegistration(127.0.0.1:40349, datanodeUuid=ba7bc4ea-b82c-4d51-b7d8-2e7d719b62fa, infoPort=46783, infoSecurePort=0, ipcPort=45535, storageInfo=lv=-57;cid=testClusterID;nsid=1174441748;c=1731761458475) 2024-11-16T12:51:01,593 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x50ba6d65eac51891 with lease ID 0xc236dc4edcb346c: from storage DS-90e24ac2-369e-4600-add3-9f2117b6171a node DatanodeRegistration(127.0.0.1:40349, datanodeUuid=ba7bc4ea-b82c-4d51-b7d8-2e7d719b62fa, infoPort=46783, infoSecurePort=0, ipcPort=45535, storageInfo=lv=-57;cid=testClusterID;nsid=1174441748;c=1731761458475), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T12:51:01,678 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2 2024-11-16T12:51:01,749 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/cluster_35bff147-7ae7-a41a-0bf2-848ec5006277/zookeeper_0, clientPort=53208, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/cluster_35bff147-7ae7-a41a-0bf2-848ec5006277/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/cluster_35bff147-7ae7-a41a-0bf2-848ec5006277/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-16T12:51:01,760 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53208 2024-11-16T12:51:01,762 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/test.cache.data in system properties and HBase conf 2024-11-16T12:51:01,762 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/hadoop.tmp.dir in system properties and HBase conf 2024-11-16T12:51:01,763 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/hadoop.log.dir in system properties and HBase conf 2024-11-16T12:51:01,763 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-16T12:51:01,763 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-16T12:51:01,764 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-16T12:51:01,773 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:01,775 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:01,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-16T12:51:01,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-16T12:51:01,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-16T12:51:01,786 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T12:51:01,786 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-16T12:51:01,786 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-16T12:51:01,786 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T12:51:01,786 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T12:51:01,787 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-16T12:51:01,787 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/nfs.dump.dir in system properties and HBase conf 2024-11-16T12:51:01,787 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/java.io.tmpdir in system properties and HBase conf 2024-11-16T12:51:01,788 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T12:51:01,788 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-16T12:51:01,788 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-16T12:51:02,074 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T12:51:02,081 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T12:51:02,085 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T12:51:02,085 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T12:51:02,086 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-16T12:51:02,087 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T12:51:02,088 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@383b4eca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/hadoop.log.dir/,AVAILABLE} 2024-11-16T12:51:02,089 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@131096ea{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T12:51:02,201 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3e6dcf12{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/java.io.tmpdir/jetty-localhost-39117-hadoop-hdfs-3_4_1-tests_jar-_-any-9127455738203854613/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T12:51:02,202 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@726b5fdd{HTTP/1.1, (http/1.1)}{localhost:39117} 2024-11-16T12:51:02,202 INFO [Time-limited test {}] server.Server(415): Started @5861ms 2024-11-16T12:51:02,443 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T12:51:02,448 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T12:51:02,449 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T12:51:02,449 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T12:51:02,449 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-16T12:51:02,450 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@13ff53d8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/hadoop.log.dir/,AVAILABLE} 2024-11-16T12:51:02,450 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@44042549{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T12:51:02,575 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@602b1375{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/java.io.tmpdir/jetty-localhost-36595-hadoop-hdfs-3_4_1-tests_jar-_-any-14382235561151756911/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T12:51:02,576 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@78ddb114{HTTP/1.1, (http/1.1)}{localhost:36595} 2024-11-16T12:51:02,576 INFO [Time-limited test {}] server.Server(415): Started @6234ms 2024-11-16T12:51:02,578 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-16T12:51:02,627 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T12:51:02,637 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T12:51:02,639 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T12:51:02,639 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T12:51:02,639 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-16T12:51:02,640 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@259b06a7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/hadoop.log.dir/,AVAILABLE} 2024-11-16T12:51:02,641 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@101ec660{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T12:51:02,742 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1a3128fc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/java.io.tmpdir/jetty-localhost-38937-hadoop-hdfs-3_4_1-tests_jar-_-any-11755116784802723657/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T12:51:02,743 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8ad07d8{HTTP/1.1, (http/1.1)}{localhost:38937} 2024-11-16T12:51:02,743 INFO [Time-limited test {}] server.Server(415): Started @6401ms 2024-11-16T12:51:02,745 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-16T12:51:03,352 WARN [Thread-205 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/cluster_35bff147-7ae7-a41a-0bf2-848ec5006277/data/data2/current/BP-1933630982-172.17.0.2-1731761461817/current, will proceed with Du for space computation calculation, 2024-11-16T12:51:03,352 WARN [Thread-204 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/cluster_35bff147-7ae7-a41a-0bf2-848ec5006277/data/data1/current/BP-1933630982-172.17.0.2-1731761461817/current, will proceed with Du for space computation calculation, 2024-11-16T12:51:03,370 WARN [Thread-168 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-16T12:51:03,374 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1e16f2f26691026f with lease ID 0xa8b987911f56b93d: Processing first storage report for DS-7ef9e9a6-250e-467c-989b-a9ffcab2299e from datanode DatanodeRegistration(127.0.0.1:36961, datanodeUuid=e55698f0-e627-47fd-9d4b-3c2a6949c890, infoPort=46667, infoSecurePort=0, ipcPort=40751, storageInfo=lv=-57;cid=testClusterID;nsid=213212349;c=1731761461817) 2024-11-16T12:51:03,375 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1e16f2f26691026f with lease ID 0xa8b987911f56b93d: from storage DS-7ef9e9a6-250e-467c-989b-a9ffcab2299e node DatanodeRegistration(127.0.0.1:36961, datanodeUuid=e55698f0-e627-47fd-9d4b-3c2a6949c890, infoPort=46667, infoSecurePort=0, ipcPort=40751, storageInfo=lv=-57;cid=testClusterID;nsid=213212349;c=1731761461817), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-16T12:51:03,375 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1e16f2f26691026f with lease ID 0xa8b987911f56b93d: Processing first storage report for DS-388232bb-11bd-4269-b50d-d0ed2aa7d6a1 from datanode DatanodeRegistration(127.0.0.1:36961, datanodeUuid=e55698f0-e627-47fd-9d4b-3c2a6949c890, infoPort=46667, infoSecurePort=0, ipcPort=40751, storageInfo=lv=-57;cid=testClusterID;nsid=213212349;c=1731761461817) 2024-11-16T12:51:03,375 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1e16f2f26691026f with lease ID 0xa8b987911f56b93d: from storage DS-388232bb-11bd-4269-b50d-d0ed2aa7d6a1 node DatanodeRegistration(127.0.0.1:36961, datanodeUuid=e55698f0-e627-47fd-9d4b-3c2a6949c890, infoPort=46667, infoSecurePort=0, ipcPort=40751, storageInfo=lv=-57;cid=testClusterID;nsid=213212349;c=1731761461817), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T12:51:03,497 WARN [Thread-216 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/cluster_35bff147-7ae7-a41a-0bf2-848ec5006277/data/data4/current/BP-1933630982-172.17.0.2-1731761461817/current, will proceed with Du for space computation calculation, 2024-11-16T12:51:03,497 WARN [Thread-215 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/cluster_35bff147-7ae7-a41a-0bf2-848ec5006277/data/data3/current/BP-1933630982-172.17.0.2-1731761461817/current, will proceed with Du for space computation calculation, 2024-11-16T12:51:03,520 WARN [Thread-191 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-16T12:51:03,525 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x24b45ad1f6b7ca9a with lease ID 0xa8b987911f56b93e: Processing first storage report for DS-06d91055-02a4-40d7-bddb-15aee81f02ac from datanode DatanodeRegistration(127.0.0.1:43257, datanodeUuid=6d7ed518-8338-4c85-a9a9-e6a32b4a2a95, infoPort=38819, infoSecurePort=0, ipcPort=39459, storageInfo=lv=-57;cid=testClusterID;nsid=213212349;c=1731761461817) 2024-11-16T12:51:03,525 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x24b45ad1f6b7ca9a with lease ID 0xa8b987911f56b93e: from storage DS-06d91055-02a4-40d7-bddb-15aee81f02ac node DatanodeRegistration(127.0.0.1:43257, datanodeUuid=6d7ed518-8338-4c85-a9a9-e6a32b4a2a95, infoPort=38819, infoSecurePort=0, ipcPort=39459, storageInfo=lv=-57;cid=testClusterID;nsid=213212349;c=1731761461817), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T12:51:03,525 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x24b45ad1f6b7ca9a with lease ID 0xa8b987911f56b93e: Processing first storage report for DS-a693babd-49d9-4b6e-a644-2d05446287ed from datanode DatanodeRegistration(127.0.0.1:43257, datanodeUuid=6d7ed518-8338-4c85-a9a9-e6a32b4a2a95, infoPort=38819, infoSecurePort=0, ipcPort=39459, storageInfo=lv=-57;cid=testClusterID;nsid=213212349;c=1731761461817) 2024-11-16T12:51:03,525 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x24b45ad1f6b7ca9a with lease ID 0xa8b987911f56b93e: from storage DS-a693babd-49d9-4b6e-a644-2d05446287ed node DatanodeRegistration(127.0.0.1:43257, datanodeUuid=6d7ed518-8338-4c85-a9a9-e6a32b4a2a95, infoPort=38819, infoSecurePort=0, ipcPort=39459, storageInfo=lv=-57;cid=testClusterID;nsid=213212349;c=1731761461817), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-16T12:51:03,591 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2 2024-11-16T12:51:03,602 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestZooKeeper#testRegionServerSessionExpired Thread=242, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=291, ProcessCount=11, AvailableMemoryMB=3685 2024-11-16T12:51:03,604 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:03,606 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:03,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741825_1001 (size=7) 2024-11-16T12:51:03,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741825_1001 (size=7) 2024-11-16T12:51:04,244 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b with version=8 2024-11-16T12:51:04,244 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/hbase-staging 2024-11-16T12:51:04,327 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-16T12:51:04,557 INFO [Time-limited test {}] client.ConnectionUtils(128): master/0450ab8807f5:0 server-side Connection retries=45 2024-11-16T12:51:04,566 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:04,567 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:04,572 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T12:51:04,572 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:04,572 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T12:51:04,695 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T12:51:04,747 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-16T12:51:04,755 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-16T12:51:04,758 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T12:51:04,780 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 83671 (auto-detected) 2024-11-16T12:51:04,781 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-16T12:51:04,798 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45941 2024-11-16T12:51:04,817 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45941 connecting to ZooKeeper ensemble=127.0.0.1:53208 2024-11-16T12:51:04,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:459410x0, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:51:04,928 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45941-0x10144fc6ca60000 connected 2024-11-16T12:51:05,002 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:05,004 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:05,013 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45941-0x10144fc6ca60000, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:51:05,017 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b, hbase.cluster.distributed=false 2024-11-16T12:51:05,044 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45941-0x10144fc6ca60000, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T12:51:05,051 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45941 2024-11-16T12:51:05,051 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45941 2024-11-16T12:51:05,052 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45941 2024-11-16T12:51:05,053 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45941 2024-11-16T12:51:05,053 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45941 2024-11-16T12:51:05,058 INFO [Time-limited test {}] client.ConnectionUtils(128): master/0450ab8807f5:0 server-side Connection retries=45 2024-11-16T12:51:05,059 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:05,059 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:05,059 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T12:51:05,060 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:05,060 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T12:51:05,060 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T12:51:05,061 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T12:51:05,062 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35663 2024-11-16T12:51:05,064 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35663 connecting to ZooKeeper ensemble=127.0.0.1:53208 2024-11-16T12:51:05,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:356630x0, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:51:05,085 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35663-0x10144fc6ca60001 connected 2024-11-16T12:51:05,098 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:05,100 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:05,103 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35663-0x10144fc6ca60001, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:51:05,104 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b, hbase.cluster.distributed=false 2024-11-16T12:51:05,107 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35663-0x10144fc6ca60001, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T12:51:05,108 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35663 2024-11-16T12:51:05,108 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35663 2024-11-16T12:51:05,108 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35663 2024-11-16T12:51:05,109 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35663 2024-11-16T12:51:05,109 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35663 2024-11-16T12:51:05,202 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0450ab8807f5:0 server-side Connection retries=45 2024-11-16T12:51:05,204 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:05,204 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:05,204 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T12:51:05,204 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:05,204 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T12:51:05,207 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T12:51:05,209 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T12:51:05,210 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41831 2024-11-16T12:51:05,212 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41831 connecting to ZooKeeper ensemble=127.0.0.1:53208 2024-11-16T12:51:05,213 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:05,215 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:05,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:418310x0, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:51:05,243 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41831-0x10144fc6ca60002 connected 2024-11-16T12:51:05,243 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41831-0x10144fc6ca60002, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:51:05,247 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T12:51:05,256 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T12:51:05,259 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41831-0x10144fc6ca60002, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:05,264 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41831-0x10144fc6ca60002, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T12:51:05,265 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41831 2024-11-16T12:51:05,266 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41831 2024-11-16T12:51:05,267 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41831 2024-11-16T12:51:05,269 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41831 2024-11-16T12:51:05,270 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41831 2024-11-16T12:51:05,323 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0450ab8807f5:0 server-side Connection retries=45 2024-11-16T12:51:05,323 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:05,324 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:05,324 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T12:51:05,324 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:05,324 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T12:51:05,324 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T12:51:05,325 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T12:51:05,326 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36743 2024-11-16T12:51:05,330 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36743 connecting to ZooKeeper ensemble=127.0.0.1:53208 2024-11-16T12:51:05,333 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:05,336 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:05,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:367430x0, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:51:05,360 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:367430x0, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:51:05,360 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36743-0x10144fc6ca60003 connected 2024-11-16T12:51:05,361 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T12:51:05,361 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T12:51:05,363 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36743-0x10144fc6ca60003, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:05,365 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36743-0x10144fc6ca60003, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T12:51:05,366 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36743 2024-11-16T12:51:05,367 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36743 2024-11-16T12:51:05,369 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36743 2024-11-16T12:51:05,370 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36743 2024-11-16T12:51:05,373 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36743 2024-11-16T12:51:05,387 DEBUG [M:1;0450ab8807f5:35663 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:1;0450ab8807f5:35663 2024-11-16T12:51:05,387 DEBUG [M:0;0450ab8807f5:45941 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0450ab8807f5:45941 2024-11-16T12:51:05,388 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0450ab8807f5,35663,1731761465058 2024-11-16T12:51:05,388 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0450ab8807f5,45941,1731761464402 2024-11-16T12:51:05,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41831-0x10144fc6ca60002, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:05,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45941-0x10144fc6ca60000, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:05,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35663-0x10144fc6ca60001, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:05,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36743-0x10144fc6ca60003, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:05,403 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45941-0x10144fc6ca60000, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0450ab8807f5,45941,1731761464402 2024-11-16T12:51:05,403 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35663-0x10144fc6ca60001, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0450ab8807f5,35663,1731761465058 2024-11-16T12:51:05,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36743-0x10144fc6ca60003, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:05,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41831-0x10144fc6ca60002, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:05,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45941-0x10144fc6ca60000, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:05,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36743-0x10144fc6ca60003, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:05,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41831-0x10144fc6ca60002, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:05,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35663-0x10144fc6ca60001, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:05,435 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45941-0x10144fc6ca60000, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T12:51:05,435 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35663-0x10144fc6ca60001, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T12:51:05,437 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0450ab8807f5,35663,1731761465058 from backup master directory 2024-11-16T12:51:05,437 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ActiveMasterManager(296): Another master is the active master, 0450ab8807f5,35663,1731761465058; waiting to become the next active master 2024-11-16T12:51:05,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35663-0x10144fc6ca60001, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0450ab8807f5,35663,1731761465058 2024-11-16T12:51:05,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41831-0x10144fc6ca60002, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:05,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45941-0x10144fc6ca60000, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:05,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36743-0x10144fc6ca60003, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:05,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35663-0x10144fc6ca60001, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:05,449 WARN [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T12:51:05,450 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0450ab8807f5,35663,1731761465058 2024-11-16T12:51:05,452 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-16T12:51:05,453 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-16T12:51:05,504 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/hbase.id] with ID: be09e536-22d9-488d-a733-7968a5bb48e1 2024-11-16T12:51:05,504 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/.tmp/hbase.id 2024-11-16T12:51:05,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741826_1002 (size=42) 2024-11-16T12:51:05,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741826_1002 (size=42) 2024-11-16T12:51:05,517 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/.tmp/hbase.id]:[hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/hbase.id] 2024-11-16T12:51:05,562 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:05,568 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T12:51:05,586 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-16T12:51:05,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45941-0x10144fc6ca60000, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:05,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41831-0x10144fc6ca60002, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:05,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35663-0x10144fc6ca60001, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:05,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36743-0x10144fc6ca60003, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:05,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741827_1003 (size=196) 2024-11-16T12:51:05,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741827_1003 (size=196) 2024-11-16T12:51:05,648 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T12:51:05,649 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T12:51:05,663 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:05,667 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:51:05,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741828_1004 (size=1189) 2024-11-16T12:51:05,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741828_1004 (size=1189) 2024-11-16T12:51:05,712 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store 2024-11-16T12:51:05,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741829_1005 (size=34) 2024-11-16T12:51:05,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741829_1005 (size=34) 2024-11-16T12:51:05,734 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-16T12:51:05,737 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:05,739 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T12:51:05,739 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:05,739 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:05,741 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T12:51:05,741 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:05,741 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:05,742 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731761465739Disabling compacts and flushes for region at 1731761465739Disabling writes for close at 1731761465741 (+2 ms)Writing region close event to WAL at 1731761465741Closed at 1731761465741 2024-11-16T12:51:05,744 WARN [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/.initializing 2024-11-16T12:51:05,744 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,35663,1731761465058 2024-11-16T12:51:05,752 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T12:51:05,765 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0450ab8807f5%2C35663%2C1731761465058, suffix=, logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,35663,1731761465058, archiveDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/oldWALs, maxLogs=10 2024-11-16T12:51:05,785 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,35663,1731761465058/0450ab8807f5%2C35663%2C1731761465058.1731761465769, exclude list is [], retry=0 2024-11-16T12:51:05,801 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43257,DS-06d91055-02a4-40d7-bddb-15aee81f02ac,DISK] 2024-11-16T12:51:05,801 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36961,DS-7ef9e9a6-250e-467c-989b-a9ffcab2299e,DISK] 2024-11-16T12:51:05,804 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-16T12:51:05,838 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,35663,1731761465058/0450ab8807f5%2C35663%2C1731761465058.1731761465769 2024-11-16T12:51:05,838 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46667:46667),(127.0.0.1/127.0.0.1:38819:38819)] 2024-11-16T12:51:05,839 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-16T12:51:05,839 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:05,843 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:05,844 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:05,882 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:05,907 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-16T12:51:05,911 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:05,913 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:05,914 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:05,918 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-16T12:51:05,918 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:05,919 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:05,920 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:05,923 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-16T12:51:05,923 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:05,925 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:05,925 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:05,928 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-16T12:51:05,929 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:05,930 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:05,930 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:05,933 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:05,934 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:05,940 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:05,941 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:05,944 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T12:51:05,947 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:05,951 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:51:05,952 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72216269, jitterRate=0.07610626518726349}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T12:51:05,960 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731761465860Initializing all the Stores at 1731761465862 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761465863 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761465863Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761465863Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761465863Cleaning up temporary data from old regions at 1731761465941 (+78 ms)Region opened successfully at 1731761465960 (+19 ms) 2024-11-16T12:51:05,961 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-16T12:51:05,992 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d1b0fd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0450ab8807f5/172.17.0.2:0 2024-11-16T12:51:06,020 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-16T12:51:06,031 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-16T12:51:06,031 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-16T12:51:06,033 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T12:51:06,035 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-16T12:51:06,040 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-16T12:51:06,040 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-16T12:51:06,060 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-16T12:51:06,068 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35663-0x10144fc6ca60001, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-16T12:51:06,117 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-16T12:51:06,120 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-16T12:51:06,122 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35663-0x10144fc6ca60001, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-16T12:51:06,133 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-16T12:51:06,136 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-16T12:51:06,142 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35663-0x10144fc6ca60001, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-16T12:51:06,150 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-16T12:51:06,153 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35663-0x10144fc6ca60001, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-16T12:51:06,158 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-16T12:51:06,177 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35663-0x10144fc6ca60001, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-16T12:51:06,183 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-16T12:51:06,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35663-0x10144fc6ca60001, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T12:51:06,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41831-0x10144fc6ca60002, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T12:51:06,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36743-0x10144fc6ca60003, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T12:51:06,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45941-0x10144fc6ca60000, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T12:51:06,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35663-0x10144fc6ca60001, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:06,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36743-0x10144fc6ca60003, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:06,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45941-0x10144fc6ca60000, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:06,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41831-0x10144fc6ca60002, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:06,195 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0450ab8807f5,35663,1731761465058, sessionid=0x10144fc6ca60001, setting cluster-up flag (Was=false) 2024-11-16T12:51:06,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35663-0x10144fc6ca60001, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:06,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45941-0x10144fc6ca60000, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:06,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36743-0x10144fc6ca60003, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:06,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41831-0x10144fc6ca60002, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:06,242 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-16T12:51:06,246 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0450ab8807f5,35663,1731761465058 2024-11-16T12:51:06,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41831-0x10144fc6ca60002, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:06,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36743-0x10144fc6ca60003, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:06,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35663-0x10144fc6ca60001, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:06,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45941-0x10144fc6ca60000, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:06,292 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-16T12:51:06,294 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0450ab8807f5,35663,1731761465058 2024-11-16T12:51:06,302 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-16T12:51:06,371 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T12:51:06,377 INFO [RS:0;0450ab8807f5:41831 {}] regionserver.HRegionServer(746): ClusterId : be09e536-22d9-488d-a733-7968a5bb48e1 2024-11-16T12:51:06,377 INFO [RS:1;0450ab8807f5:36743 {}] regionserver.HRegionServer(746): ClusterId : be09e536-22d9-488d-a733-7968a5bb48e1 2024-11-16T12:51:06,379 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-16T12:51:06,379 DEBUG [RS:0;0450ab8807f5:41831 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T12:51:06,379 DEBUG [RS:1;0450ab8807f5:36743 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T12:51:06,384 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0450ab8807f5,35663,1731761465058 Number of backup masters: 1 0450ab8807f5,45941,1731761464402 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-16T12:51:06,412 DEBUG [RS:0;0450ab8807f5:41831 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T12:51:06,412 DEBUG [RS:1;0450ab8807f5:36743 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T12:51:06,412 DEBUG [RS:0;0450ab8807f5:41831 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T12:51:06,412 DEBUG [RS:1;0450ab8807f5:36743 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T12:51:06,413 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:06,413 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:06,413 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:06,413 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:06,414 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0450ab8807f5:0, corePoolSize=10, maxPoolSize=10 2024-11-16T12:51:06,414 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:06,414 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0450ab8807f5:0, corePoolSize=2, maxPoolSize=2 2024-11-16T12:51:06,414 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:06,417 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731761496417 2024-11-16T12:51:06,419 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-16T12:51:06,420 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-16T12:51:06,421 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T12:51:06,422 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-16T12:51:06,424 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-16T12:51:06,425 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-16T12:51:06,425 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-16T12:51:06,425 DEBUG [RS:0;0450ab8807f5:41831 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T12:51:06,425 DEBUG [RS:1;0450ab8807f5:36743 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T12:51:06,425 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-16T12:51:06,426 DEBUG [RS:0;0450ab8807f5:41831 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ed7dd84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0450ab8807f5/172.17.0.2:0 2024-11-16T12:51:06,426 DEBUG [RS:1;0450ab8807f5:36743 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36a12904, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0450ab8807f5/172.17.0.2:0 2024-11-16T12:51:06,426 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:06,429 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-16T12:51:06,429 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:06,430 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-16T12:51:06,430 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-16T12:51:06,431 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-16T12:51:06,434 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-16T12:51:06,434 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-16T12:51:06,436 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.large.0-1731761466436,5,FailOnTimeoutGroup] 2024-11-16T12:51:06,437 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.small.0-1731761466436,5,FailOnTimeoutGroup] 2024-11-16T12:51:06,437 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:06,437 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-16T12:51:06,438 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:06,439 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:06,440 DEBUG [RS:0;0450ab8807f5:41831 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0450ab8807f5:41831 2024-11-16T12:51:06,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741831_1007 (size=1321) 2024-11-16T12:51:06,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741831_1007 (size=1321) 2024-11-16T12:51:06,443 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-16T12:51:06,443 DEBUG [RS:1;0450ab8807f5:36743 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;0450ab8807f5:36743 2024-11-16T12:51:06,443 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:06,444 INFO [RS:0;0450ab8807f5:41831 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T12:51:06,444 INFO [RS:1;0450ab8807f5:36743 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T12:51:06,444 INFO [RS:0;0450ab8807f5:41831 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T12:51:06,444 INFO [RS:1;0450ab8807f5:36743 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T12:51:06,444 DEBUG [RS:0;0450ab8807f5:41831 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T12:51:06,444 DEBUG [RS:1;0450ab8807f5:36743 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T12:51:06,447 INFO [RS:1;0450ab8807f5:36743 {}] regionserver.HRegionServer(2659): reportForDuty to master=0450ab8807f5,35663,1731761465058 with port=36743, startcode=1731761465323 2024-11-16T12:51:06,447 INFO [RS:0;0450ab8807f5:41831 {}] regionserver.HRegionServer(2659): reportForDuty to master=0450ab8807f5,35663,1731761465058 with port=41831, startcode=1731761465169 2024-11-16T12:51:06,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741832_1008 (size=32) 2024-11-16T12:51:06,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741832_1008 (size=32) 2024-11-16T12:51:06,457 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:06,460 DEBUG [RS:0;0450ab8807f5:41831 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T12:51:06,460 DEBUG [RS:1;0450ab8807f5:36743 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T12:51:06,460 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T12:51:06,463 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T12:51:06,463 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:06,464 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:06,464 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T12:51:06,467 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T12:51:06,467 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:06,468 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:06,468 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T12:51:06,471 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T12:51:06,471 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:06,472 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:06,473 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T12:51:06,476 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T12:51:06,476 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:06,477 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:06,477 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T12:51:06,479 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740 2024-11-16T12:51:06,480 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740 2024-11-16T12:51:06,483 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T12:51:06,483 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T12:51:06,484 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T12:51:06,487 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T12:51:06,491 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:51:06,492 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70768113, jitterRate=0.05452705919742584}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T12:51:06,494 INFO [HMaster-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37541, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T12:51:06,494 INFO [HMaster-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35293, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T12:51:06,498 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731761466458Initializing all the Stores at 1731761466460 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761466460Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761466460Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761466460Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761466460Cleaning up temporary data from old regions at 1731761466483 (+23 ms)Region opened successfully at 1731761466498 (+15 ms) 2024-11-16T12:51:06,498 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T12:51:06,498 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T12:51:06,499 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T12:51:06,499 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T12:51:06,499 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T12:51:06,499 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35663 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0450ab8807f5,36743,1731761465323 2024-11-16T12:51:06,500 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T12:51:06,500 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731761466498Disabling compacts and flushes for region at 1731761466498Disabling writes for close at 1731761466499 (+1 ms)Writing region close event to WAL at 1731761466500 (+1 ms)Closed at 1731761466500 2024-11-16T12:51:06,502 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35663 {}] master.ServerManager(517): Registering regionserver=0450ab8807f5,36743,1731761465323 2024-11-16T12:51:06,504 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T12:51:06,504 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-16T12:51:06,510 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35663 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0450ab8807f5,41831,1731761465169 2024-11-16T12:51:06,511 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35663 {}] master.ServerManager(517): Registering regionserver=0450ab8807f5,41831,1731761465169 2024-11-16T12:51:06,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T12:51:06,514 DEBUG [RS:1;0450ab8807f5:36743 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:06,514 DEBUG [RS:0;0450ab8807f5:41831 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:06,514 DEBUG [RS:1;0450ab8807f5:36743 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44975 2024-11-16T12:51:06,514 DEBUG [RS:0;0450ab8807f5:41831 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44975 2024-11-16T12:51:06,514 DEBUG [RS:0;0450ab8807f5:41831 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T12:51:06,514 DEBUG [RS:1;0450ab8807f5:36743 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T12:51:06,521 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T12:51:06,524 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-16T12:51:06,570 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35663-0x10144fc6ca60001, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T12:51:06,570 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45941-0x10144fc6ca60000, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T12:51:06,591 DEBUG [RS:0;0450ab8807f5:41831 {}] zookeeper.ZKUtil(111): regionserver:41831-0x10144fc6ca60002, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0450ab8807f5,41831,1731761465169 2024-11-16T12:51:06,591 WARN [RS:0;0450ab8807f5:41831 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T12:51:06,591 INFO [RS:0;0450ab8807f5:41831 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:51:06,592 DEBUG [RS:1;0450ab8807f5:36743 {}] zookeeper.ZKUtil(111): regionserver:36743-0x10144fc6ca60003, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0450ab8807f5,36743,1731761465323 2024-11-16T12:51:06,592 WARN [RS:1;0450ab8807f5:36743 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T12:51:06,592 DEBUG [RS:0;0450ab8807f5:41831 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169 2024-11-16T12:51:06,592 INFO [RS:1;0450ab8807f5:36743 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:51:06,592 DEBUG [RS:1;0450ab8807f5:36743 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,36743,1731761465323 2024-11-16T12:51:06,594 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0450ab8807f5,36743,1731761465323] 2024-11-16T12:51:06,594 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0450ab8807f5,41831,1731761465169] 2024-11-16T12:51:06,618 INFO [RS:1;0450ab8807f5:36743 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T12:51:06,618 INFO [RS:0;0450ab8807f5:41831 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T12:51:06,633 INFO [RS:0;0450ab8807f5:41831 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T12:51:06,633 INFO [RS:1;0450ab8807f5:36743 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T12:51:06,637 INFO [RS:1;0450ab8807f5:36743 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T12:51:06,637 INFO [RS:0;0450ab8807f5:41831 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T12:51:06,637 INFO [RS:1;0450ab8807f5:36743 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:06,638 INFO [RS:0;0450ab8807f5:41831 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:06,638 INFO [RS:1;0450ab8807f5:36743 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T12:51:06,638 INFO [RS:0;0450ab8807f5:41831 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T12:51:06,643 INFO [RS:1;0450ab8807f5:36743 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T12:51:06,643 INFO [RS:0;0450ab8807f5:41831 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T12:51:06,645 INFO [RS:1;0450ab8807f5:36743 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:06,645 INFO [RS:0;0450ab8807f5:41831 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:06,645 DEBUG [RS:1;0450ab8807f5:36743 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:06,645 DEBUG [RS:0;0450ab8807f5:41831 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:06,645 DEBUG [RS:1;0450ab8807f5:36743 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:06,645 DEBUG [RS:0;0450ab8807f5:41831 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:06,645 DEBUG [RS:1;0450ab8807f5:36743 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:06,645 DEBUG [RS:0;0450ab8807f5:41831 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:06,645 DEBUG [RS:1;0450ab8807f5:36743 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:06,645 DEBUG [RS:1;0450ab8807f5:36743 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:06,645 DEBUG [RS:0;0450ab8807f5:41831 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:06,645 DEBUG [RS:1;0450ab8807f5:36743 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0, corePoolSize=2, maxPoolSize=2 2024-11-16T12:51:06,645 DEBUG [RS:0;0450ab8807f5:41831 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:06,646 DEBUG [RS:1;0450ab8807f5:36743 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:06,646 DEBUG [RS:0;0450ab8807f5:41831 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0, corePoolSize=2, maxPoolSize=2 2024-11-16T12:51:06,646 DEBUG [RS:1;0450ab8807f5:36743 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:06,646 DEBUG [RS:1;0450ab8807f5:36743 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:06,646 DEBUG [RS:0;0450ab8807f5:41831 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:06,646 DEBUG [RS:1;0450ab8807f5:36743 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:06,646 DEBUG [RS:0;0450ab8807f5:41831 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:06,646 DEBUG [RS:1;0450ab8807f5:36743 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:06,646 DEBUG [RS:1;0450ab8807f5:36743 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:06,646 DEBUG [RS:0;0450ab8807f5:41831 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:06,646 DEBUG [RS:1;0450ab8807f5:36743 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:51:06,646 DEBUG [RS:0;0450ab8807f5:41831 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:06,646 DEBUG [RS:1;0450ab8807f5:36743 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:51:06,646 DEBUG [RS:0;0450ab8807f5:41831 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:06,646 DEBUG [RS:0;0450ab8807f5:41831 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:06,647 DEBUG [RS:0;0450ab8807f5:41831 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:51:06,647 DEBUG [RS:0;0450ab8807f5:41831 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:51:06,648 INFO [RS:1;0450ab8807f5:36743 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:06,648 INFO [RS:1;0450ab8807f5:36743 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:06,648 INFO [RS:0;0450ab8807f5:41831 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:06,648 INFO [RS:1;0450ab8807f5:36743 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:06,648 INFO [RS:0;0450ab8807f5:41831 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:06,648 INFO [RS:1;0450ab8807f5:36743 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:06,648 INFO [RS:0;0450ab8807f5:41831 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:06,648 INFO [RS:0;0450ab8807f5:41831 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:06,648 INFO [RS:1;0450ab8807f5:36743 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:06,648 INFO [RS:0;0450ab8807f5:41831 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:06,648 INFO [RS:1;0450ab8807f5:36743 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,36743,1731761465323-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T12:51:06,649 INFO [RS:0;0450ab8807f5:41831 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,41831,1731761465169-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T12:51:06,666 INFO [RS:1;0450ab8807f5:36743 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T12:51:06,666 INFO [RS:0;0450ab8807f5:41831 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T12:51:06,668 INFO [RS:0;0450ab8807f5:41831 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,41831,1731761465169-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:06,668 INFO [RS:1;0450ab8807f5:36743 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,36743,1731761465323-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:06,669 INFO [RS:1;0450ab8807f5:36743 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:06,669 INFO [RS:0;0450ab8807f5:41831 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:06,669 INFO [RS:1;0450ab8807f5:36743 {}] regionserver.Replication(171): 0450ab8807f5,36743,1731761465323 started 2024-11-16T12:51:06,669 INFO [RS:0;0450ab8807f5:41831 {}] regionserver.Replication(171): 0450ab8807f5,41831,1731761465169 started 2024-11-16T12:51:06,675 WARN [0450ab8807f5:35663 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-16T12:51:06,686 INFO [RS:1;0450ab8807f5:36743 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:06,686 INFO [RS:1;0450ab8807f5:36743 {}] regionserver.HRegionServer(1482): Serving as 0450ab8807f5,36743,1731761465323, RpcServer on 0450ab8807f5/172.17.0.2:36743, sessionid=0x10144fc6ca60003 2024-11-16T12:51:06,687 DEBUG [RS:1;0450ab8807f5:36743 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T12:51:06,687 DEBUG [RS:1;0450ab8807f5:36743 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0450ab8807f5,36743,1731761465323 2024-11-16T12:51:06,687 DEBUG [RS:1;0450ab8807f5:36743 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,36743,1731761465323' 2024-11-16T12:51:06,688 DEBUG [RS:1;0450ab8807f5:36743 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T12:51:06,689 DEBUG [RS:1;0450ab8807f5:36743 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T12:51:06,689 DEBUG [RS:1;0450ab8807f5:36743 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T12:51:06,689 DEBUG [RS:1;0450ab8807f5:36743 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T12:51:06,690 DEBUG [RS:1;0450ab8807f5:36743 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0450ab8807f5,36743,1731761465323 2024-11-16T12:51:06,690 DEBUG [RS:1;0450ab8807f5:36743 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,36743,1731761465323' 2024-11-16T12:51:06,690 DEBUG [RS:1;0450ab8807f5:36743 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T12:51:06,690 DEBUG [RS:1;0450ab8807f5:36743 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T12:51:06,691 DEBUG [RS:1;0450ab8807f5:36743 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T12:51:06,691 INFO [RS:1;0450ab8807f5:36743 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T12:51:06,691 INFO [RS:0;0450ab8807f5:41831 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:06,691 INFO [RS:1;0450ab8807f5:36743 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T12:51:06,691 INFO [RS:0;0450ab8807f5:41831 {}] regionserver.HRegionServer(1482): Serving as 0450ab8807f5,41831,1731761465169, RpcServer on 0450ab8807f5/172.17.0.2:41831, sessionid=0x10144fc6ca60002 2024-11-16T12:51:06,692 DEBUG [RS:0;0450ab8807f5:41831 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T12:51:06,692 DEBUG [RS:0;0450ab8807f5:41831 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0450ab8807f5,41831,1731761465169 2024-11-16T12:51:06,692 DEBUG [RS:0;0450ab8807f5:41831 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,41831,1731761465169' 2024-11-16T12:51:06,692 DEBUG [RS:0;0450ab8807f5:41831 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T12:51:06,692 DEBUG [RS:0;0450ab8807f5:41831 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T12:51:06,693 DEBUG [RS:0;0450ab8807f5:41831 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T12:51:06,693 DEBUG [RS:0;0450ab8807f5:41831 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T12:51:06,693 DEBUG [RS:0;0450ab8807f5:41831 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0450ab8807f5,41831,1731761465169 2024-11-16T12:51:06,693 DEBUG [RS:0;0450ab8807f5:41831 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,41831,1731761465169' 2024-11-16T12:51:06,693 DEBUG [RS:0;0450ab8807f5:41831 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T12:51:06,694 DEBUG [RS:0;0450ab8807f5:41831 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T12:51:06,695 DEBUG [RS:0;0450ab8807f5:41831 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T12:51:06,695 INFO [RS:0;0450ab8807f5:41831 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T12:51:06,695 INFO [RS:0;0450ab8807f5:41831 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T12:51:06,797 INFO [RS:1;0450ab8807f5:36743 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T12:51:06,797 INFO [RS:0;0450ab8807f5:41831 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T12:51:06,802 INFO [RS:0;0450ab8807f5:41831 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0450ab8807f5%2C41831%2C1731761465169, suffix=, logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169, archiveDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/oldWALs, maxLogs=32 2024-11-16T12:51:06,802 INFO [RS:1;0450ab8807f5:36743 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0450ab8807f5%2C36743%2C1731761465323, suffix=, logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,36743,1731761465323, archiveDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/oldWALs, maxLogs=32 2024-11-16T12:51:06,824 DEBUG [RS:1;0450ab8807f5:36743 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,36743,1731761465323/0450ab8807f5%2C36743%2C1731761465323.1731761466805, exclude list is [], retry=0 2024-11-16T12:51:06,824 DEBUG [RS:0;0450ab8807f5:41831 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169/0450ab8807f5%2C41831%2C1731761465169.1731761466805, exclude list is [], retry=0 2024-11-16T12:51:06,829 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43257,DS-06d91055-02a4-40d7-bddb-15aee81f02ac,DISK] 2024-11-16T12:51:06,829 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36961,DS-7ef9e9a6-250e-467c-989b-a9ffcab2299e,DISK] 2024-11-16T12:51:06,829 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36961,DS-7ef9e9a6-250e-467c-989b-a9ffcab2299e,DISK] 2024-11-16T12:51:06,830 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43257,DS-06d91055-02a4-40d7-bddb-15aee81f02ac,DISK] 2024-11-16T12:51:06,832 INFO [RS:1;0450ab8807f5:36743 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,36743,1731761465323/0450ab8807f5%2C36743%2C1731761465323.1731761466805 2024-11-16T12:51:06,833 DEBUG [RS:1;0450ab8807f5:36743 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:46667:46667)] 2024-11-16T12:51:06,833 INFO [RS:0;0450ab8807f5:41831 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169/0450ab8807f5%2C41831%2C1731761465169.1731761466805 2024-11-16T12:51:06,833 DEBUG [RS:0;0450ab8807f5:41831 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46667:46667),(127.0.0.1/127.0.0.1:38819:38819)] 2024-11-16T12:51:06,931 DEBUG [0450ab8807f5:35663 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-16T12:51:06,941 DEBUG [0450ab8807f5:35663 {}] balancer.BalancerClusterState(204): Hosts are {0450ab8807f5=0} racks are {/default-rack=0} 2024-11-16T12:51:06,947 DEBUG [0450ab8807f5:35663 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T12:51:06,947 DEBUG [0450ab8807f5:35663 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T12:51:06,947 DEBUG [0450ab8807f5:35663 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T12:51:06,947 DEBUG [0450ab8807f5:35663 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T12:51:06,947 INFO [0450ab8807f5:35663 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T12:51:06,947 INFO [0450ab8807f5:35663 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T12:51:06,947 DEBUG [0450ab8807f5:35663 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T12:51:06,954 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0450ab8807f5,41831,1731761465169 2024-11-16T12:51:06,959 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0450ab8807f5,41831,1731761465169, state=OPENING 2024-11-16T12:51:06,999 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-16T12:51:07,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36743-0x10144fc6ca60003, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:07,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45941-0x10144fc6ca60000, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:07,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35663-0x10144fc6ca60001, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:07,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41831-0x10144fc6ca60002, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:07,018 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:07,019 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:07,019 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:07,020 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:07,022 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T12:51:07,025 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0450ab8807f5,41831,1731761465169}] 2024-11-16T12:51:07,200 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T12:51:07,202 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47993, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T12:51:07,213 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T12:51:07,213 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:51:07,214 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T12:51:07,217 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0450ab8807f5%2C41831%2C1731761465169.meta, suffix=.meta, logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169, archiveDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/oldWALs, maxLogs=32 2024-11-16T12:51:07,233 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169/0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta, exclude list is [], retry=0 2024-11-16T12:51:07,236 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43257,DS-06d91055-02a4-40d7-bddb-15aee81f02ac,DISK] 2024-11-16T12:51:07,236 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36961,DS-7ef9e9a6-250e-467c-989b-a9ffcab2299e,DISK] 2024-11-16T12:51:07,240 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169/0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta 2024-11-16T12:51:07,240 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46667:46667),(127.0.0.1/127.0.0.1:38819:38819)] 2024-11-16T12:51:07,241 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T12:51:07,242 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T12:51:07,245 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T12:51:07,250 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T12:51:07,255 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T12:51:07,256 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:07,256 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T12:51:07,256 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T12:51:07,259 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T12:51:07,261 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T12:51:07,261 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:07,261 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:07,262 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T12:51:07,263 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T12:51:07,263 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:07,264 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:07,264 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T12:51:07,266 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T12:51:07,266 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:07,267 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:07,267 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T12:51:07,269 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T12:51:07,269 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:07,270 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:07,270 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T12:51:07,271 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740 2024-11-16T12:51:07,274 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740 2024-11-16T12:51:07,277 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T12:51:07,277 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T12:51:07,278 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T12:51:07,280 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T12:51:07,282 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60751554, jitterRate=-0.09473130106925964}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T12:51:07,283 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T12:51:07,284 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731761467256Writing region info on filesystem at 1731761467256Initializing all the Stores at 1731761467259 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761467259Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761467259Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761467259Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761467259Cleaning up temporary data from old regions at 1731761467277 (+18 ms)Running coprocessor post-open hooks at 1731761467283 (+6 ms)Region opened successfully at 1731761467284 (+1 ms) 2024-11-16T12:51:07,290 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731761467192 2024-11-16T12:51:07,301 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T12:51:07,301 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T12:51:07,303 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0450ab8807f5,41831,1731761465169 2024-11-16T12:51:07,306 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0450ab8807f5,41831,1731761465169, state=OPEN 2024-11-16T12:51:07,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45941-0x10144fc6ca60000, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:07,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41831-0x10144fc6ca60002, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:07,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35663-0x10144fc6ca60001, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:07,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36743-0x10144fc6ca60003, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:07,333 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:07,333 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:07,333 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:07,333 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:07,334 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0450ab8807f5,41831,1731761465169 2024-11-16T12:51:07,343 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-16T12:51:07,343 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0450ab8807f5,41831,1731761465169 in 310 msec 2024-11-16T12:51:07,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-16T12:51:07,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 835 msec 2024-11-16T12:51:07,352 DEBUG [PEWorker-4 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T12:51:07,353 INFO [PEWorker-4 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-16T12:51:07,372 DEBUG [PEWorker-4 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T12:51:07,373 DEBUG [PEWorker-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1] 2024-11-16T12:51:07,392 DEBUG [PEWorker-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T12:51:07,394 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43087, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T12:51:07,412 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0780 sec 2024-11-16T12:51:07,412 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731761467412, completionTime=-1 2024-11-16T12:51:07,415 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-16T12:51:07,415 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T12:51:07,440 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T12:51:07,440 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731761527440 2024-11-16T12:51:07,440 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731761587440 2024-11-16T12:51:07,440 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-16T12:51:07,441 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-16T12:51:07,471 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,35663,1731761465058-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:07,471 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,35663,1731761465058-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:07,471 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,35663,1731761465058-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:07,473 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0450ab8807f5:35663, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:07,474 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:07,477 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:07,481 DEBUG [master/0450ab8807f5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-16T12:51:07,506 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.056sec 2024-11-16T12:51:07,508 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-16T12:51:07,509 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-16T12:51:07,510 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-16T12:51:07,510 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-16T12:51:07,510 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-16T12:51:07,511 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,35663,1731761465058-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T12:51:07,511 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,35663,1731761465058-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-16T12:51:07,519 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-16T12:51:07,520 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-16T12:51:07,520 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,35663,1731761465058-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:07,586 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x38112a7b to 127.0.0.1:53208 with session timeout=1000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6f334f3e 2024-11-16T12:51:07,586 WARN [Time-limited test {}] client.ZKConnectionRegistry(87): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry 2024-11-16T12:51:07,617 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66ec6f97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T12:51:07,621 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-16T12:51:07,621 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-16T12:51:07,626 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T12:51:07,630 DEBUG [ReadOnlyZKClient-127.0.0.1:53208@0x38112a7b {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1] 2024-11-16T12:51:07,631 DEBUG [ReadOnlyZKClient-127.0.0.1:53208@0x38112a7b {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T12:51:07,638 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51562, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T12:51:07,658 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0450ab8807f5,35663,1731761465058 2024-11-16T12:51:07,659 INFO [Time-limited test {}] hbase.TestZooKeeper(120): Starting testRegionServerSessionExpired 2024-11-16T12:51:07,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45941-0x10144fc6ca60000, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T12:51:07,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35663-0x10144fc6ca60001, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T12:51:07,684 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0450ab8807f5,41831,1731761465169] 2024-11-16T12:51:07,688 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35663 {}] master.ServerManager(480): Server REPORT rejected; currently processing 0450ab8807f5,41831,1731761465169 as dead server 2024-11-16T12:51:07,688 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35663 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.YouAreDeadException: Server REPORT rejected; currently processing 0450ab8807f5,41831,1731761465169 as dead server at org.apache.hadoop.hbase.master.ServerManager.checkIsDead(ServerManager.java:481) ~[classes/:?] at org.apache.hadoop.hbase.master.ServerManager.regionServerReport(ServerManager.java:318) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:643) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:07,693 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35663 {}] ipc.CallRunner(138): callId: 10 service: RegionServerStatusService methodName: RegionServerReport size: 1.5 K connection: 172.17.0.2:35293 deadline: 1731761477687, exception=org.apache.hadoop.hbase.YouAreDeadException: Server REPORT rejected; currently processing 0450ab8807f5,41831,1731761465169 as dead server 2024-11-16T12:51:07,698 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0450ab8807f5,41831,1731761465169 already deleted, retry=false 2024-11-16T12:51:07,698 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 0450ab8807f5,41831,1731761465169 on 0450ab8807f5,35663,1731761465058 2024-11-16T12:51:07,704 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 0450ab8807f5,41831,1731761465169, splitWal=true, meta=true 2024-11-16T12:51:07,706 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=4 for 0450ab8807f5,41831,1731761465169 (carryingMeta=true) 0450ab8807f5,41831,1731761465169/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@67198d01[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-16T12:51:07,707 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=4, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 0450ab8807f5,41831,1731761465169, splitWal=true, meta=true 2024-11-16T12:51:07,709 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=4, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 0450ab8807f5,41831,1731761465169, splitWal=true, meta=true, isMeta: true 2024-11-16T12:51:07,712 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting 2024-11-16T12:51:07,714 INFO [PEWorker-2 {}] master.SplitWALManager(105): 0450ab8807f5,41831,1731761465169 WAL count=1, meta=true 2024-11-16T12:51:07,716 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta}] 2024-11-16T12:51:07,719 ERROR [RS:0;0450ab8807f5:41831 {}] regionserver.HRegionServer(2440): ***** ABORTING region server 0450ab8807f5,41831,1731761465169: org.apache.hadoop.hbase.YouAreDeadException: Server REPORT rejected; currently processing 0450ab8807f5,41831,1731761465169 as dead server at org.apache.hadoop.hbase.master.ServerManager.checkIsDead(ServerManager.java:481) at org.apache.hadoop.hbase.master.ServerManager.regionServerReport(ServerManager.java:318) at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:643) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ***** org.apache.hadoop.hbase.YouAreDeadException: org.apache.hadoop.hbase.YouAreDeadException: Server REPORT rejected; currently processing 0450ab8807f5,41831,1731761465169 as dead server at org.apache.hadoop.hbase.master.ServerManager.checkIsDead(ServerManager.java:481) at org.apache.hadoop.hbase.master.ServerManager.regionServerReport(ServerManager.java:318) at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:643) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.tryRegionServerReport(HRegionServer.java:1082) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:902) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) ~[test-classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) ~[test-classes/:?] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) ~[test-classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.YouAreDeadException: Server REPORT rejected; currently processing 0450ab8807f5,41831,1731761465169 as dead server at org.apache.hadoop.hbase.master.ServerManager.checkIsDead(ServerManager.java:481) at org.apache.hadoop.hbase.master.ServerManager.regionServerReport(ServerManager.java:318) at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:643) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-16T12:51:07,720 ERROR [RS:0;0450ab8807f5:41831 {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-16T12:51:07,723 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=0450ab8807f5,36743,1731761465323 2024-11-16T12:51:07,724 DEBUG [RS:0;0450ab8807f5:41831 {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-16T12:51:07,726 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta, worker=0450ab8807f5,36743,1731761465323}] 2024-11-16T12:51:07,727 DEBUG [RS:0;0450ab8807f5:41831 {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-16T12:51:07,727 DEBUG [RS:0;0450ab8807f5:41831 {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-16T12:51:07,727 DEBUG [RS:0;0450ab8807f5:41831 {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-16T12:51:07,728 INFO [RS:0;0450ab8807f5:41831 {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 204920064 }, "NonHeapMemoryUsage": { "committed": 137953280, "init": 7667712, "max": -1, "used": 135803016 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "0450ab8807f5" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "0450ab8807f5" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "0450ab8807f5" } ] } 2024-11-16T12:51:07,733 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35663 {}] master.MasterRpcServices(700): 0450ab8807f5,41831,1731761465169 reported a fatal error: ***** ABORTING region server 0450ab8807f5,41831,1731761465169: org.apache.hadoop.hbase.YouAreDeadException: Server REPORT rejected; currently processing 0450ab8807f5,41831,1731761465169 as dead server at org.apache.hadoop.hbase.master.ServerManager.checkIsDead(ServerManager.java:481) at org.apache.hadoop.hbase.master.ServerManager.regionServerReport(ServerManager.java:318) at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:643) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ***** Cause: org.apache.hadoop.hbase.YouAreDeadException: org.apache.hadoop.hbase.YouAreDeadException: Server REPORT rejected; currently processing 0450ab8807f5,41831,1731761465169 as dead server at org.apache.hadoop.hbase.master.ServerManager.checkIsDead(ServerManager.java:481) at org.apache.hadoop.hbase.master.ServerManager.regionServerReport(ServerManager.java:318) at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:643) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) at org.apache.hadoop.hbase.regionserver.HRegionServer.tryRegionServerReport(HRegionServer.java:1082) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:902) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.YouAreDeadException): org.apache.hadoop.hbase.YouAreDeadException: Server REPORT rejected; currently processing 0450ab8807f5,41831,1731761465169 as dead server at org.apache.hadoop.hbase.master.ServerManager.checkIsDead(ServerManager.java:481) at org.apache.hadoop.hbase.master.ServerManager.regionServerReport(ServerManager.java:318) at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:643) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-16T12:51:07,736 INFO [RS:0;0450ab8807f5:41831 {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0450ab8807f5,41831,1731761465169' ***** 2024-11-16T12:51:07,736 INFO [RS:0;0450ab8807f5:41831 {}] regionserver.HRegionServer(2210): STOPPED: org.apache.hadoop.hbase.YouAreDeadException: Server REPORT rejected; currently processing 0450ab8807f5,41831,1731761465169 as dead server at org.apache.hadoop.hbase.master.ServerManager.checkIsDead(ServerManager.java:481) at org.apache.hadoop.hbase.master.ServerManager.regionServerReport(ServerManager.java:318) at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:643) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T12:51:07,736 INFO [RS:0;0450ab8807f5:41831 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T12:51:07,737 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T12:51:07,737 INFO [RS:0;0450ab8807f5:41831 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-16T12:51:07,737 INFO [RS:0;0450ab8807f5:41831 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-16T12:51:07,737 INFO [RS:0;0450ab8807f5:41831 {}] regionserver.HRegionServer(956): aborting server 0450ab8807f5,41831,1731761465169 2024-11-16T12:51:07,737 INFO [RS:0;0450ab8807f5:41831 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T12:51:07,737 INFO [RS:0;0450ab8807f5:41831 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0450ab8807f5:41831. 2024-11-16T12:51:07,738 DEBUG [RS:0;0450ab8807f5:41831 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:51:07,743 DEBUG [RS:0;0450ab8807f5:41831 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:07,744 INFO [RS:0;0450ab8807f5:41831 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T12:51:07,744 INFO [RS:0;0450ab8807f5:41831 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T12:51:07,744 INFO [RS:0;0450ab8807f5:41831 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T12:51:07,744 INFO [RS:0;0450ab8807f5:41831 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T12:51:07,745 INFO [RS:0;0450ab8807f5:41831 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-16T12:51:07,745 DEBUG [RS:0;0450ab8807f5:41831 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-16T12:51:07,745 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T12:51:07,746 DEBUG [RS:0;0450ab8807f5:41831 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T12:51:07,746 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T12:51:07,746 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T12:51:07,746 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T12:51:07,746 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T12:51:07,746 ERROR [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 74 in region hbase:meta,,1.1588230740 2024-11-16T12:51:07,747 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T12:51:07,747 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T12:51:07,747 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731761467745Running coprocessor pre-close hooks at 1731761467745Disabling compacts and flushes for region at 1731761467745Disabling writes for close at 1731761467746 (+1 ms)Writing region close event to WAL at 1731761467746Running coprocessor post-close hooks at 1731761467747 (+1 ms)Closed at 1731761467747 2024-11-16T12:51:07,747 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T12:51:07,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41831-0x10144fc6ca60002, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Disconnected, path=null 2024-11-16T12:51:07,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41831-0x10144fc6ca60002, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Disconnected, path=null 2024-11-16T12:51:07,762 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(640): regionserver:41831-0x10144fc6ca60002, quorum=127.0.0.1:53208, baseZNode=/hbase Received Disconnected from ZooKeeper, ignoring 2024-11-16T12:51:07,762 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(640): regionserver:41831-0x10144fc6ca60002, quorum=127.0.0.1:53208, baseZNode=/hbase Received Disconnected from ZooKeeper, ignoring 2024-11-16T12:51:07,769 INFO [regionserver/0450ab8807f5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:51:07,784 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2501): ZK Closed Session 0x10144fc6ca60002 2024-11-16T12:51:07,891 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T12:51:07,894 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38563, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T12:51:07,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36743 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=6 2024-11-16T12:51:07,919 INFO [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0 {event_type=RS_LOG_REPLAY, pid=6}] wal.WALSplitter(299): Splitting hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting/0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta, size=0 (0bytes) 2024-11-16T12:51:07,919 WARN [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0 {event_type=RS_LOG_REPLAY, pid=6}] wal.WALSplitter(453): File hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting/0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta might be still open, length is 0 2024-11-16T12:51:07,920 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0 {event_type=RS_LOG_REPLAY, pid=6}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-16T12:51:07,920 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0 {event_type=RS_LOG_REPLAY, pid=6}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-16T12:51:07,920 INFO [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0 {event_type=RS_LOG_REPLAY, pid=6}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting/0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta 2024-11-16T12:51:07,922 WARN [IPC Server handler 0 on default port 44975 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting/0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta has not been closed. Lease recovery is in progress. RecoveryId = 1012 for block blk_1073741835_1011 2024-11-16T12:51:07,925 INFO [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0 {event_type=RS_LOG_REPLAY, pid=6}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting/0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta after 4ms 2024-11-16T12:51:07,946 INFO [RS:0;0450ab8807f5:41831 {}] regionserver.HRegionServer(976): stopping server 0450ab8807f5,41831,1731761465169; all regions closed. 2024-11-16T12:51:07,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741835_1011 (size=488) 2024-11-16T12:51:07,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741835_1011 (size=488) 2024-11-16T12:51:07,957 WARN [Close-WAL-Writer-0 {}] wal.AsyncProtobufLogWriter(165): normal close failed, try recover org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException: Client (=DFSClient_NONMAPREDUCE_676520379_22) is not the lease owner (=DFSClient_NONMAPREDUCE_735921662_22: /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting/0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta (inode 16440) [Lease. Holder: DFSClient_NONMAPREDUCE_676520379_22, pending creates: 1] at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3210) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:163) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: Client (=DFSClient_NONMAPREDUCE_676520379_22) is not the lease owner (=DFSClient_NONMAPREDUCE_735921662_22: /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting/0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta (inode 16440) [Lease. Holder: DFSClient_NONMAPREDUCE_676520379_22, pending creates: 1] at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3210) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 6 more 2024-11-16T12:51:07,959 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169/0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta 2024-11-16T12:51:07,960 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169/0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.recoverLease(FSNamesystem.java:2869) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.recoverLease(NameNodeRpcServer.java:872) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.recoverLease(ClientNamenodeProtocolServerSideTranslatorPB.java:834) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:951) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.recoverAndClose(FanOutOneBlockAsyncDFSOutput.java:605) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:166) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169/0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.recoverLease(FSNamesystem.java:2869) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.recoverLease(NameNodeRpcServer.java:872) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.recoverLease(ClientNamenodeProtocolServerSideTranslatorPB.java:834) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.recoverLease(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$recoverLease$24(ClientNamenodeProtocolTranslatorPB.java:685) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.recoverLease(ClientNamenodeProtocolTranslatorPB.java:685) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:949) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 17 more 2024-11-16T12:51:07,961 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169/0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta 2024-11-16T12:51:07,962 ERROR [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2033): Unable to recover lease after several attempts. Give up. java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169/0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.recoverLease(FSNamesystem.java:2869) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.recoverLease(NameNodeRpcServer.java:872) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.recoverLease(ClientNamenodeProtocolServerSideTranslatorPB.java:834) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:951) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169/0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.recoverLease(FSNamesystem.java:2869) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.recoverLease(NameNodeRpcServer.java:872) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.recoverLease(ClientNamenodeProtocolServerSideTranslatorPB.java:834) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.recoverLease(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$recoverLease$24(ClientNamenodeProtocolTranslatorPB.java:685) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.recoverLease(ClientNamenodeProtocolTranslatorPB.java:685) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:949) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 16 more 2024-11-16T12:51:07,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741833_1009 (size=93) 2024-11-16T12:51:07,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741833_1009 (size=93) 2024-11-16T12:51:07,968 DEBUG [RS:0;0450ab8807f5:41831 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:07,968 INFO [RS:0;0450ab8807f5:41831 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:51:07,968 INFO [RS:0;0450ab8807f5:41831 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T12:51:07,968 INFO [RS:0;0450ab8807f5:41831 {}] hbase.ChoreService(370): Chore service for: regionserver/0450ab8807f5:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T12:51:07,969 INFO [RS:0;0450ab8807f5:41831 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T12:51:07,969 INFO [regionserver/0450ab8807f5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T12:51:07,969 INFO [RS:0;0450ab8807f5:41831 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41831 2024-11-16T12:51:08,835 INFO [Time-limited test-EventThread {}] hbase.HBaseTestingUtil$1(2484): Monitor ZKW received event=WatchedEvent state:Expired type:None path:null 2024-11-16T12:51:08,937 INFO [Time-limited test {}] hbase.TestZooKeeper(153): Creating table testRegionServerSessionExpired_1731761468936 2024-11-16T12:51:08,947 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T12:51:08,948 DEBUG [ReadOnlyZKClient-127.0.0.1:53208@0x38112a7b {}] client.AsyncConnectionImpl(321): The fetched master address is 0450ab8807f5,35663,1731761465058 2024-11-16T12:51:08,951 DEBUG [ReadOnlyZKClient-127.0.0.1:53208@0x38112a7b {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@786d91f9 2024-11-16T12:51:08,952 DEBUG [ReadOnlyZKClient-127.0.0.1:53208@0x38112a7b {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T12:51:08,954 INFO [HMaster-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T12:51:08,959 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35663 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRegionServerSessionExpired_1731761468936', {NAME => 'fam', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T12:51:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35663 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRegionServerSessionExpired_1731761468936 2024-11-16T12:51:08,968 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRegionServerSessionExpired_1731761468936 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T12:51:08,970 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:08,972 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35663 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRegionServerSessionExpired_1731761468936" procId is: 7 2024-11-16T12:51:08,972 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRegionServerSessionExpired_1731761468936 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T12:51:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35663 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T12:51:08,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741836_1013 (size=395) 2024-11-16T12:51:08,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741836_1013 (size=395) 2024-11-16T12:51:08,987 INFO [RegionOpenAndInit-testRegionServerSessionExpired_1731761468936-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 289b79361cdd89dea41ef5c11f08967b, NAME => 'testRegionServerSessionExpired_1731761468936,,1731761468956.289b79361cdd89dea41ef5c11f08967b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionServerSessionExpired_1731761468936', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'fam', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:08,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741837_1014 (size=79) 2024-11-16T12:51:08,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741837_1014 (size=79) 2024-11-16T12:51:08,997 DEBUG [RegionOpenAndInit-testRegionServerSessionExpired_1731761468936-pool-0 {}] regionserver.HRegion(898): Instantiated testRegionServerSessionExpired_1731761468936,,1731761468956.289b79361cdd89dea41ef5c11f08967b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:08,997 DEBUG [RegionOpenAndInit-testRegionServerSessionExpired_1731761468936-pool-0 {}] regionserver.HRegion(1722): Closing 289b79361cdd89dea41ef5c11f08967b, disabling compactions & flushes 2024-11-16T12:51:08,997 INFO [RegionOpenAndInit-testRegionServerSessionExpired_1731761468936-pool-0 {}] regionserver.HRegion(1755): Closing region testRegionServerSessionExpired_1731761468936,,1731761468956.289b79361cdd89dea41ef5c11f08967b. 2024-11-16T12:51:08,997 DEBUG [RegionOpenAndInit-testRegionServerSessionExpired_1731761468936-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionServerSessionExpired_1731761468936,,1731761468956.289b79361cdd89dea41ef5c11f08967b. 2024-11-16T12:51:08,997 DEBUG [RegionOpenAndInit-testRegionServerSessionExpired_1731761468936-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRegionServerSessionExpired_1731761468936,,1731761468956.289b79361cdd89dea41ef5c11f08967b. after waiting 0 ms 2024-11-16T12:51:08,997 DEBUG [RegionOpenAndInit-testRegionServerSessionExpired_1731761468936-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRegionServerSessionExpired_1731761468936,,1731761468956.289b79361cdd89dea41ef5c11f08967b. 2024-11-16T12:51:08,998 INFO [RegionOpenAndInit-testRegionServerSessionExpired_1731761468936-pool-0 {}] regionserver.HRegion(1973): Closed testRegionServerSessionExpired_1731761468936,,1731761468956.289b79361cdd89dea41ef5c11f08967b. 2024-11-16T12:51:08,998 DEBUG [RegionOpenAndInit-testRegionServerSessionExpired_1731761468936-pool-0 {}] regionserver.HRegion(1676): Region close journal for 289b79361cdd89dea41ef5c11f08967b: Waiting for close lock at 1731761468997Disabling compacts and flushes for region at 1731761468997Disabling writes for close at 1731761468997Writing region close event to WAL at 1731761468997Closed at 1731761468997 2024-11-16T12:51:09,000 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRegionServerSessionExpired_1731761468936 execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T12:51:09,004 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRegionServerSessionExpired_1731761468936,,1731761468956.289b79361cdd89dea41ef5c11f08967b.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731761469000"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731761469000"}]},"ts":"1731761469000"} 2024-11-16T12:51:09,006 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 0450ab8807f5:41831 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0450ab8807f5/172.17.0.2:41831 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:09,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1, error=java.net.ConnectException: Call to address=0450ab8807f5:41831 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0450ab8807f5/172.17.0.2:41831 2024-11-16T12:51:09,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T12:51:09,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1 from cache 2024-11-16T12:51:09,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 0450ab8807f5:41831 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0450ab8807f5/172.17.0.2:41831 2024-11-16T12:51:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35663 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T12:51:09,119 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T12:51:09,119 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1] 2024-11-16T12:51:09,119 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 0450ab8807f5:41831 this server is in the failed servers list 2024-11-16T12:51:09,120 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=0450ab8807f5:41831 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 0450ab8807f5:41831 2024-11-16T12:51:09,120 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 0450ab8807f5:41831 2024-11-16T12:51:09,120 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1 from cache 2024-11-16T12:51:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35663 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T12:51:09,329 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T12:51:09,329 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1] 2024-11-16T12:51:09,330 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 0450ab8807f5:41831 this server is in the failed servers list 2024-11-16T12:51:09,330 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=0450ab8807f5:41831 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 0450ab8807f5:41831 2024-11-16T12:51:09,331 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 0450ab8807f5:41831 2024-11-16T12:51:09,331 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1 from cache 2024-11-16T12:51:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35663 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T12:51:09,639 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T12:51:09,639 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1] 2024-11-16T12:51:09,639 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 0450ab8807f5:41831 this server is in the failed servers list 2024-11-16T12:51:09,640 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=0450ab8807f5:41831 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 0450ab8807f5:41831 2024-11-16T12:51:09,640 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 0450ab8807f5:41831 2024-11-16T12:51:09,640 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1 from cache 2024-11-16T12:51:09,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41831-0x10144fc6ca60002, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Expired, path=null 2024-11-16T12:51:09,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41831-0x10144fc6ca60002, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Expired, path=null 2024-11-16T12:51:09,714 DEBUG [zk-event-processor-pool-0 {}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: regionserver:41831-0x10144fc6ca60002, quorum=127.0.0.1:53208, baseZNode=/hbase regionserver:41831-0x10144fc6ca60002 received expired from ZooKeeper, aborting 2024-11-16T12:51:09,714 DEBUG [zk-event-processor-pool-0 {}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: regionserver:41831-0x10144fc6ca60002, quorum=127.0.0.1:53208, baseZNode=/hbase regionserver:41831-0x10144fc6ca60002 received expired from ZooKeeper, aborting 2024-11-16T12:51:09,814 WARN [RS:0;0450ab8807f5:41831 {}] regionserver.HRegionServer(1023): Failed deleting my ephemeral node org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /hbase/rs/0450ab8807f5,41831,1731761465169 at org.apache.zookeeper.KeeperException.create(KeeperException.java:134) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.delete(ZooKeeper.java:1614) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.delete(RecoverableZooKeeper.java:201) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.deleteNode(ZKUtil.java:851) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.deleteNode(ZKUtil.java:843) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.deleteMyEphemeralNode(HRegionServer.java:1516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:1019) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) ~[test-classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) ~[test-classes/:?] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) ~[test-classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:09,814 INFO [RS:0;0450ab8807f5:41831 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T12:51:09,814 INFO [RS:0;0450ab8807f5:41831 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T12:51:09,814 INFO [RS:0;0450ab8807f5:41831 {}] regionserver.HRegionServer(1031): Exiting; stopping=0450ab8807f5,41831,1731761465169; zookeeper connection closed. 2024-11-16T12:51:09,815 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@574d3e26 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@574d3e26 2024-11-16T12:51:10,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35663 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T12:51:10,149 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T12:51:10,150 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1] 2024-11-16T12:51:10,151 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 0450ab8807f5:41831 this server is in the failed servers list 2024-11-16T12:51:10,151 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=0450ab8807f5:41831 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 0450ab8807f5:41831 2024-11-16T12:51:10,151 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 0450ab8807f5:41831 2024-11-16T12:51:10,152 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1 from cache 2024-11-16T12:51:11,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35663 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T12:51:11,159 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T12:51:11,159 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1] 2024-11-16T12:51:11,160 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 0450ab8807f5:41831 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0450ab8807f5/172.17.0.2:41831 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:11,163 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 0450ab8807f5,41831,1731761465169 for 'testRegionServerSessionExpired_1731761468936,,1731761468956.289b79361cdd89dea41ef5c11f08967b.' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 1200000 ms, time elapsed = 2155 ms java.net.ConnectException: Call to address=0450ab8807f5:41831 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0450ab8807f5/172.17.0.2:41831 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0450ab8807f5/172.17.0.2:41831 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T12:51:11,163 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1, error=java.net.ConnectException: Call to address=0450ab8807f5:41831 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0450ab8807f5/172.17.0.2:41831 2024-11-16T12:51:11,163 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T12:51:11,163 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1 from cache 2024-11-16T12:51:11,164 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 0450ab8807f5:41831 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0450ab8807f5/172.17.0.2:41831 2024-11-16T12:51:11,927 INFO [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0 {event_type=RS_LOG_REPLAY, pid=6}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting/0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta after 4007ms 2024-11-16T12:51:11,933 INFO [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0 {event_type=RS_LOG_REPLAY, pid=6}] wal.WALSplitter(310): Open hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting/0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta took 4015ms 2024-11-16T12:51:11,942 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0 {event_type=RS_LOG_REPLAY, pid=6}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 18446744073709551615 } 2024-11-16T12:51:11,943 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0 {event_type=RS_LOG_REPLAY, pid=6}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting/0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta so closing down 2024-11-16T12:51:11,943 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0 {event_type=RS_LOG_REPLAY, pid=6}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T12:51:11,947 INFO [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta.temp 2024-11-16T12:51:11,950 INFO [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta.temp 2024-11-16T12:51:11,951 INFO [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0 {event_type=RS_LOG_REPLAY, pid=6}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T12:51:11,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741838_1015 (size=247) 2024-11-16T12:51:11,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741838_1015 (size=247) 2024-11-16T12:51:11,961 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-16T12:51:11,963 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta.temp to hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740/recovered.edits/0000000000000000005 2024-11-16T12:51:11,967 INFO [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0 {event_type=RS_LOG_REPLAY, pid=6}] wal.WALSplitter(425): Processed 3 edits across 1 Regions in 29 ms; skipped=1; WAL=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting/0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta, size=0, length=0, corrupted=false, cancelled=false 2024-11-16T12:51:11,967 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0 {event_type=RS_LOG_REPLAY, pid=6}] wal.WALSplitter(428): Completed split of hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting/0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta, journal: Splitting hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting/0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta, size=0 (0bytes) at 1731761467919Finishing writing output for hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting/0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta so closing down at 1731761471943 (+4024 ms)Creating recovered edits writer path=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta.temp at 1731761471950 (+7 ms)3 split writer threads finished at 1731761471951 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731761471961 (+10 ms)Rename recovered edits hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta.temp to hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740/recovered.edits/0000000000000000005 at 1731761471963 (+2 ms)Processed 3 edits across 1 Regions in 29 ms; skipped=1; WAL=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting/0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta, size=0, length=0, corrupted=false, cancelled=false at 1731761471967 (+4 ms) 2024-11-16T12:51:11,967 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0 {event_type=RS_LOG_REPLAY, pid=6}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting/0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta 2024-11-16T12:51:11,968 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0 {event_type=RS_LOG_REPLAY, pid=6}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=6 2024-11-16T12:51:11,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35663 {}] master.HMaster(4169): Remote procedure done, pid=6 2024-11-16T12:51:11,978 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting/0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta to hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/oldWALs 2024-11-16T12:51:11,982 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-16T12:51:11,983 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta, worker=0450ab8807f5,36743,1731761465323 in 4.2520 sec 2024-11-16T12:51:11,985 DEBUG [PEWorker-2 {}] master.SplitWALManager(172): Release split WAL worker=0450ab8807f5,36743,1731761465323 2024-11-16T12:51:11,990 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-16T12:51:11,990 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; SplitWALProcedure 0450ab8807f5%2C41831%2C1731761465169.meta.1731761467219.meta, worker=0450ab8807f5,36743,1731761465323 in 4.2700 sec 2024-11-16T12:51:11,992 INFO [PEWorker-3 {}] master.SplitLogManager(171): hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting dir is empty, no logs to split. 2024-11-16T12:51:11,992 INFO [PEWorker-3 {}] master.SplitWALManager(105): 0450ab8807f5,41831,1731761465169 WAL count=0, meta=true 2024-11-16T12:51:11,993 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if 0450ab8807f5,41831,1731761465169 WAL splitting is done? wals=0, meta=true 2024-11-16T12:51:11,995 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T12:51:11,997 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T12:51:11,999 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T12:51:12,150 DEBUG [0450ab8807f5:35663 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-16T12:51:12,150 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0450ab8807f5,36743,1731761465323 2024-11-16T12:51:12,153 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0450ab8807f5,36743,1731761465323, state=OPENING 2024-11-16T12:51:12,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35663-0x10144fc6ca60001, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:12,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36743-0x10144fc6ca60003, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:12,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45941-0x10144fc6ca60000, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:12,183 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T12:51:12,183 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:12,183 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:12,183 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:12,183 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0450ab8807f5,36743,1731761465323}] 2024-11-16T12:51:12,347 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T12:51:12,347 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:51:12,348 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T12:51:12,351 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0450ab8807f5%2C36743%2C1731761465323.meta, suffix=.meta, logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,36743,1731761465323, archiveDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/oldWALs, maxLogs=32 2024-11-16T12:51:12,366 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,36743,1731761465323/0450ab8807f5%2C36743%2C1731761465323.meta.1731761472354.meta, exclude list is [], retry=0 2024-11-16T12:51:12,370 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43257,DS-06d91055-02a4-40d7-bddb-15aee81f02ac,DISK] 2024-11-16T12:51:12,370 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36961,DS-7ef9e9a6-250e-467c-989b-a9ffcab2299e,DISK] 2024-11-16T12:51:12,372 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,36743,1731761465323/0450ab8807f5%2C36743%2C1731761465323.meta.1731761472354.meta 2024-11-16T12:51:12,373 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:46667:46667)] 2024-11-16T12:51:12,373 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T12:51:12,373 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T12:51:12,374 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T12:51:12,374 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T12:51:12,374 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T12:51:12,374 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:12,374 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T12:51:12,374 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T12:51:12,376 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T12:51:12,378 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T12:51:12,378 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:12,379 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:12,379 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T12:51:12,380 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T12:51:12,380 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:12,381 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:12,381 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T12:51:12,382 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T12:51:12,382 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:12,383 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:12,383 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T12:51:12,385 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T12:51:12,385 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:12,386 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:12,386 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T12:51:12,388 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740 2024-11-16T12:51:12,391 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740 2024-11-16T12:51:12,393 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740/recovered.edits/0000000000000000005 2024-11-16T12:51:12,400 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=5, path=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740/recovered.edits/0000000000000000005 2024-11-16T12:51:12,404 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB 2024-11-16T12:51:12,452 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740/.tmp/ns/dfb7061ac4ab423ba7cee5337abd5ce4 is 43, key is default/ns:d/1731761467398/Put/seqid=0 2024-11-16T12:51:12,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741840_1017 (size=5153) 2024-11-16T12:51:12,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741840_1017 (size=5153) 2024-11-16T12:51:12,464 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740/.tmp/ns/dfb7061ac4ab423ba7cee5337abd5ce4 2024-11-16T12:51:12,503 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740/.tmp/ns/dfb7061ac4ab423ba7cee5337abd5ce4 as hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740/ns/dfb7061ac4ab423ba7cee5337abd5ce4 2024-11-16T12:51:12,512 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740/ns/dfb7061ac4ab423ba7cee5337abd5ce4, entries=2, sequenceid=5, filesize=5.0 K 2024-11-16T12:51:12,517 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 111ms, sequenceid=5, compaction requested=false; wal=null 2024-11-16T12:51:12,518 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-16T12:51:12,521 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740/recovered.edits/0000000000000000005 2024-11-16T12:51:12,523 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T12:51:12,523 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T12:51:12,524 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T12:51:12,526 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T12:51:12,530 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-16T12:51:12,531 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69777213, jitterRate=0.0397614985704422}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T12:51:12,531 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T12:51:12,533 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731761472374Writing region info on filesystem at 1731761472374Initializing all the Stores at 1731761472376 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761472376Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761472376Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761472376Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761472376Obtaining lock to block concurrent updates at 1731761472404 (+28 ms)Preparing flush snapshotting stores in 1588230740 at 1731761472405 (+1 ms)Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1731761472407 (+2 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731761472407Flushing 1588230740/ns: creating writer at 1731761472409 (+2 ms)Flushing 1588230740/ns: appending metadata at 1731761472443 (+34 ms)Flushing 1588230740/ns: closing flushed file at 1731761472445 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4535a3fa: reopening flushed file at 1731761472501 (+56 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 111ms, sequenceid=5, compaction requested=false; wal=null at 1731761472517 (+16 ms)Cleaning up temporary data from old regions at 1731761472523 (+6 ms)Running coprocessor post-open hooks at 1731761472531 (+8 ms)Region opened successfully at 1731761472532 (+1 ms) 2024-11-16T12:51:12,534 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1731761472339 2024-11-16T12:51:12,537 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T12:51:12,537 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T12:51:12,538 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=6, regionLocation=0450ab8807f5,36743,1731761465323 2024-11-16T12:51:12,540 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0450ab8807f5,36743,1731761465323, state=OPEN 2024-11-16T12:51:12,559 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35663 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=0450ab8807f5,36743,1731761465323, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-16T12:51:12,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35663-0x10144fc6ca60001, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:12,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36743-0x10144fc6ca60003, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:12,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45941-0x10144fc6ca60000, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:12,581 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0450ab8807f5,36743,1731761465323 2024-11-16T12:51:12,582 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:12,582 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:12,582 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:12,589 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-16T12:51:12,590 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0450ab8807f5,36743,1731761465323 in 399 msec 2024-11-16T12:51:12,594 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=4 2024-11-16T12:51:12,594 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 594 msec 2024-11-16T12:51:12,595 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(207): 0450ab8807f5,41831,1731761465169 had 1 regions 2024-11-16T12:51:12,597 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=4, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 0450ab8807f5,41831,1731761465169, splitWal=true, meta=true, isMeta: false 2024-11-16T12:51:12,600 INFO [PEWorker-4 {}] master.SplitWALManager(105): 0450ab8807f5,41831,1731761465169 WAL count=1, meta=false 2024-11-16T12:51:12,600 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=4, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 0450ab8807f5%2C41831%2C1731761465169.1731761466805}] 2024-11-16T12:51:12,603 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=0450ab8807f5,36743,1731761465323 2024-11-16T12:51:12,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 0450ab8807f5%2C41831%2C1731761465169.1731761466805, worker=0450ab8807f5,36743,1731761465323}] 2024-11-16T12:51:12,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36743 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=11 2024-11-16T12:51:12,771 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T12:51:12,785 INFO [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-1 {event_type=RS_LOG_REPLAY, pid=11}] wal.WALSplitter(299): Splitting hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting/0450ab8807f5%2C41831%2C1731761465169.1731761466805, size=93 (93bytes) 2024-11-16T12:51:12,785 INFO [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-1 {event_type=RS_LOG_REPLAY, pid=11}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting/0450ab8807f5%2C41831%2C1731761465169.1731761466805 2024-11-16T12:51:12,786 INFO [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-1 {event_type=RS_LOG_REPLAY, pid=11}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting/0450ab8807f5%2C41831%2C1731761465169.1731761466805 after 1ms 2024-11-16T12:51:12,790 INFO [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-1 {event_type=RS_LOG_REPLAY, pid=11}] wal.WALSplitter(310): Open hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting/0450ab8807f5%2C41831%2C1731761465169.1731761466805 took 6ms 2024-11-16T12:51:12,793 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-1 {event_type=RS_LOG_REPLAY, pid=11}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting/0450ab8807f5%2C41831%2C1731761465169.1731761466805 so closing down 2024-11-16T12:51:12,793 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-1 {event_type=RS_LOG_REPLAY, pid=11}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T12:51:12,794 INFO [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-1 {event_type=RS_LOG_REPLAY, pid=11}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T12:51:12,794 INFO [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-1 {event_type=RS_LOG_REPLAY, pid=11}] wal.WALSplitter(425): Processed 0 edits across 0 Regions in 1 ms; skipped=0; WAL=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting/0450ab8807f5%2C41831%2C1731761465169.1731761466805, size=93, length=93, corrupted=false, cancelled=false 2024-11-16T12:51:12,794 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-1 {event_type=RS_LOG_REPLAY, pid=11}] wal.WALSplitter(428): Completed split of hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting/0450ab8807f5%2C41831%2C1731761465169.1731761466805, journal: Splitting hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting/0450ab8807f5%2C41831%2C1731761465169.1731761466805, size=93 (93bytes) at 1731761472785Finishing writing output for hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting/0450ab8807f5%2C41831%2C1731761465169.1731761466805 so closing down at 1731761472793 (+8 ms)3 split writer threads finished at 1731761472794 (+1 ms)Processed 0 edits across 0 Regions in 1 ms; skipped=0; WAL=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting/0450ab8807f5%2C41831%2C1731761465169.1731761466805, size=93, length=93, corrupted=false, cancelled=false at 1731761472794 2024-11-16T12:51:12,794 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-1 {event_type=RS_LOG_REPLAY, pid=11}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting/0450ab8807f5%2C41831%2C1731761465169.1731761466805 2024-11-16T12:51:12,794 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-1 {event_type=RS_LOG_REPLAY, pid=11}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=11 2024-11-16T12:51:12,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35663 {}] master.HMaster(4169): Remote procedure done, pid=11 2024-11-16T12:51:12,801 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting/0450ab8807f5%2C41831%2C1731761465169.1731761466805 to hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/oldWALs 2024-11-16T12:51:12,805 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-16T12:51:12,806 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 0450ab8807f5%2C41831%2C1731761465169.1731761466805, worker=0450ab8807f5,36743,1731761465323 in 197 msec 2024-11-16T12:51:12,808 DEBUG [PEWorker-4 {}] master.SplitWALManager(172): Release split WAL worker=0450ab8807f5,36743,1731761465323 2024-11-16T12:51:12,812 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=4 2024-11-16T12:51:12,812 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; SplitWALProcedure 0450ab8807f5%2C41831%2C1731761465169.1731761466805, worker=0450ab8807f5,36743,1731761465323 in 208 msec 2024-11-16T12:51:12,815 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,41831,1731761465169-splitting dir is empty, no logs to split. 2024-11-16T12:51:12,815 INFO [PEWorker-2 {}] master.SplitWALManager(105): 0450ab8807f5,41831,1731761465169 WAL count=0, meta=false 2024-11-16T12:51:12,815 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 0450ab8807f5,41831,1731761465169 WAL splitting is done? wals=0, meta=false 2024-11-16T12:51:12,820 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(291): removed crashed server 0450ab8807f5,41831,1731761465169 after splitting done 2024-11-16T12:51:12,824 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; ServerCrashProcedure 0450ab8807f5,41831,1731761465169, splitWal=true, meta=true in 5.1210 sec 2024-11-16T12:51:13,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35663 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T12:51:13,169 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T12:51:13,169 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,36743,1731761465323, seqNum=-1] 2024-11-16T12:51:13,169 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T12:51:13,172 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59693, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T12:51:13,176 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T12:51:13,178 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRegionServerSessionExpired_1731761468936 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T12:51:13,183 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRegionServerSessionExpired_1731761468936","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731761473178"}]},"ts":"1731761473178"} 2024-11-16T12:51:13,187 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRegionServerSessionExpired_1731761468936, state=ENABLING in hbase:meta 2024-11-16T12:51:13,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionServerSessionExpired_1731761468936, region=289b79361cdd89dea41ef5c11f08967b, ASSIGN}] 2024-11-16T12:51:13,190 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionServerSessionExpired_1731761468936, region=289b79361cdd89dea41ef5c11f08967b, ASSIGN 2024-11-16T12:51:13,192 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRegionServerSessionExpired_1731761468936, region=289b79361cdd89dea41ef5c11f08967b, ASSIGN; state=OFFLINE, location=0450ab8807f5,36743,1731761465323; forceNewPlan=false, retain=false 2024-11-16T12:51:13,345 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=289b79361cdd89dea41ef5c11f08967b, regionState=OPENING, regionLocation=0450ab8807f5,36743,1731761465323 2024-11-16T12:51:13,355 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRegionServerSessionExpired_1731761468936, region=289b79361cdd89dea41ef5c11f08967b, ASSIGN because future has completed 2024-11-16T12:51:13,356 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 289b79361cdd89dea41ef5c11f08967b, server=0450ab8807f5,36743,1731761465323}] 2024-11-16T12:51:13,523 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open testRegionServerSessionExpired_1731761468936,,1731761468956.289b79361cdd89dea41ef5c11f08967b. 2024-11-16T12:51:13,523 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => 289b79361cdd89dea41ef5c11f08967b, NAME => 'testRegionServerSessionExpired_1731761468936,,1731761468956.289b79361cdd89dea41ef5c11f08967b.', STARTKEY => '', ENDKEY => ''} 2024-11-16T12:51:13,524 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRegionServerSessionExpired_1731761468936 289b79361cdd89dea41ef5c11f08967b 2024-11-16T12:51:13,524 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated testRegionServerSessionExpired_1731761468936,,1731761468956.289b79361cdd89dea41ef5c11f08967b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:13,524 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for 289b79361cdd89dea41ef5c11f08967b 2024-11-16T12:51:13,524 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for 289b79361cdd89dea41ef5c11f08967b 2024-11-16T12:51:13,527 INFO [StoreOpener-289b79361cdd89dea41ef5c11f08967b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family fam of region 289b79361cdd89dea41ef5c11f08967b 2024-11-16T12:51:13,530 INFO [StoreOpener-289b79361cdd89dea41ef5c11f08967b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 289b79361cdd89dea41ef5c11f08967b columnFamilyName fam 2024-11-16T12:51:13,530 DEBUG [StoreOpener-289b79361cdd89dea41ef5c11f08967b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:13,531 INFO [StoreOpener-289b79361cdd89dea41ef5c11f08967b-1 {}] regionserver.HStore(327): Store=289b79361cdd89dea41ef5c11f08967b/fam, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:13,531 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for 289b79361cdd89dea41ef5c11f08967b 2024-11-16T12:51:13,533 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionServerSessionExpired_1731761468936/289b79361cdd89dea41ef5c11f08967b 2024-11-16T12:51:13,534 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionServerSessionExpired_1731761468936/289b79361cdd89dea41ef5c11f08967b 2024-11-16T12:51:13,535 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for 289b79361cdd89dea41ef5c11f08967b 2024-11-16T12:51:13,535 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for 289b79361cdd89dea41ef5c11f08967b 2024-11-16T12:51:13,539 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for 289b79361cdd89dea41ef5c11f08967b 2024-11-16T12:51:13,542 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionServerSessionExpired_1731761468936/289b79361cdd89dea41ef5c11f08967b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:51:13,542 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened 289b79361cdd89dea41ef5c11f08967b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68499028, jitterRate=0.020715057849884033}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T12:51:13,542 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 289b79361cdd89dea41ef5c11f08967b 2024-11-16T12:51:13,543 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for 289b79361cdd89dea41ef5c11f08967b: Running coprocessor pre-open hook at 1731761473524Writing region info on filesystem at 1731761473524Initializing all the Stores at 1731761473526 (+2 ms)Instantiating store for column family {NAME => 'fam', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761473526Cleaning up temporary data from old regions at 1731761473535 (+9 ms)Running coprocessor post-open hooks at 1731761473543 (+8 ms)Region opened successfully at 1731761473543 2024-11-16T12:51:13,545 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for testRegionServerSessionExpired_1731761468936,,1731761468956.289b79361cdd89dea41ef5c11f08967b., pid=13, masterSystemTime=1731761473510 2024-11-16T12:51:13,548 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for testRegionServerSessionExpired_1731761468936,,1731761468956.289b79361cdd89dea41ef5c11f08967b. 2024-11-16T12:51:13,548 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened testRegionServerSessionExpired_1731761468936,,1731761468956.289b79361cdd89dea41ef5c11f08967b. 2024-11-16T12:51:13,549 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=289b79361cdd89dea41ef5c11f08967b, regionState=OPEN, openSeqNum=2, regionLocation=0450ab8807f5,36743,1731761465323 2024-11-16T12:51:13,552 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 289b79361cdd89dea41ef5c11f08967b, server=0450ab8807f5,36743,1731761465323 because future has completed 2024-11-16T12:51:13,557 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=12 2024-11-16T12:51:13,558 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 289b79361cdd89dea41ef5c11f08967b, server=0450ab8807f5,36743,1731761465323 in 198 msec 2024-11-16T12:51:13,561 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=7 2024-11-16T12:51:13,561 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRegionServerSessionExpired_1731761468936, region=289b79361cdd89dea41ef5c11f08967b, ASSIGN in 369 msec 2024-11-16T12:51:13,562 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRegionServerSessionExpired_1731761468936 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T12:51:13,562 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRegionServerSessionExpired_1731761468936","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731761473562"}]},"ts":"1731761473562"} 2024-11-16T12:51:13,565 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testRegionServerSessionExpired_1731761468936, state=ENABLED in hbase:meta 2024-11-16T12:51:13,567 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRegionServerSessionExpired_1731761468936 execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T12:51:13,570 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRegionServerSessionExpired_1731761468936 in 4.6050 sec 2024-11-16T12:51:14,744 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionServerSessionExpired_1731761468936 2024-11-16T12:51:14,745 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionServerSessionExpired_1731761468936 Metrics about Tables on a single HBase RegionServer 2024-11-16T12:51:14,748 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-16T12:51:14,748 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-16T12:51:14,750 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T12:51:14,750 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-16T12:51:14,751 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-16T12:51:14,751 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-16T12:51:17,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35663 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T12:51:17,170 INFO [Time-limited test {}] hbase.TestZooKeeper(164): Putting table testRegionServerSessionExpired_1731761468936 2024-11-16T12:51:17,170 INFO [RPCClient-NioEventLoopGroup-6-2 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRegionServerSessionExpired_1731761468936 completed 2024-11-16T12:51:17,176 WARN [RPCClient-NioEventLoopGroup-6-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 0450ab8807f5:41831 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 0450ab8807f5/172.17.0.2:41831 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:17,177 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1, error=java.net.ConnectException: Call to address=0450ab8807f5:41831 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 0450ab8807f5/172.17.0.2:41831 2024-11-16T12:51:17,177 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1 is java.net.ConnectException: Connection refused 2024-11-16T12:51:17,177 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,41831,1731761465169, seqNum=-1 from cache 2024-11-16T12:51:17,178 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.FailedServers(52): Added failed server with address 0450ab8807f5:41831 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 0450ab8807f5/172.17.0.2:41831 2024-11-16T12:51:17,289 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T12:51:17,294 DEBUG [ReadOnlyZKClient-127.0.0.1:53208@0x38112a7b {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,36743,1731761465323, seqNum=-1] 2024-11-16T12:51:17,295 DEBUG [ReadOnlyZKClient-127.0.0.1:53208@0x38112a7b {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T12:51:17,300 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T12:51:17,311 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRegionServerSessionExpired_1731761468936', row='testrow', locateType=CURRENT is [region=testRegionServerSessionExpired_1731761468936,,1731761468956.289b79361cdd89dea41ef5c11f08967b., hostname=0450ab8807f5,36743,1731761465323, seqNum=2] 2024-11-16T12:51:17,316 ERROR [Time-limited test {}] master.HMaster(3225): ***** ABORTING master 0450ab8807f5,45941,1731761464402: killAll ***** 2024-11-16T12:51:17,316 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master '0450ab8807f5,45941,1731761464402' ***** 2024-11-16T12:51:17,316 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: Stopped by Time-limited test 2024-11-16T12:51:17,316 INFO [M:0;0450ab8807f5:45941 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T12:51:17,316 INFO [M:0;0450ab8807f5:45941 {}] hbase.ChoreService(370): Chore service for: master/0450ab8807f5:0 had [] on shutdown 2024-11-16T12:51:17,316 INFO [M:0;0450ab8807f5:45941 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T12:51:17,316 DEBUG [M:0;0450ab8807f5:45941 {}] master.HMaster(1795): Stopping service threads 2024-11-16T12:51:17,317 ERROR [Time-limited test {}] master.HMaster(3218): Master server abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-16T12:51:17,317 ERROR [Time-limited test {}] master.HMaster(3225): ***** ABORTING master 0450ab8807f5,35663,1731761465058: killAll ***** 2024-11-16T12:51:17,317 INFO [M:0;0450ab8807f5:45941 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45941 2024-11-16T12:51:17,317 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master '0450ab8807f5,35663,1731761465058' ***** 2024-11-16T12:51:17,317 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: Stopped by Time-limited test 2024-11-16T12:51:17,317 INFO [M:1;0450ab8807f5:35663 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T12:51:17,317 INFO [M:0;0450ab8807f5:45941 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T12:51:17,317 INFO [M:1;0450ab8807f5:35663 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:1;0450ab8807f5:35663. 2024-11-16T12:51:17,317 DEBUG [M:1;0450ab8807f5:35663 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:51:17,318 DEBUG [M:1;0450ab8807f5:35663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:17,318 INFO [M:1;0450ab8807f5:35663 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T12:51:17,318 DEBUG [M:1;0450ab8807f5:35663 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T12:51:17,318 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T12:51:17,318 DEBUG [M:1;0450ab8807f5:35663 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T12:51:17,318 DEBUG [master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.small.0-1731761466436 {}] cleaner.HFileCleaner(306): Exit Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.small.0-1731761466436,5,FailOnTimeoutGroup] 2024-11-16T12:51:17,318 DEBUG [master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.large.0-1731761466436 {}] cleaner.HFileCleaner(306): Exit Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.large.0-1731761466436,5,FailOnTimeoutGroup] 2024-11-16T12:51:17,318 INFO [M:1;0450ab8807f5:35663 {}] hbase.ChoreService(370): Chore service for: master/0450ab8807f5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T12:51:17,318 INFO [M:1;0450ab8807f5:35663 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T12:51:17,318 DEBUG [M:1;0450ab8807f5:35663 {}] master.HMaster(1795): Stopping service threads 2024-11-16T12:51:17,318 INFO [M:1;0450ab8807f5:35663 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T12:51:17,319 INFO [M:1;0450ab8807f5:35663 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T12:51:17,319 INFO [M:1;0450ab8807f5:35663 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=true 2024-11-16T12:51:17,319 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T12:51:17,346 DEBUG [HMaster-EventLoopGroup-3-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0450ab8807f5,35663,1731761465058 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:17,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35663-0x10144fc6ca60001, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:17,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36743-0x10144fc6ca60003, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:17,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45941-0x10144fc6ca60000, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0450ab8807f5,45941,1731761464402 2024-11-16T12:51:17,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45941-0x10144fc6ca60000, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:17,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35663-0x10144fc6ca60001, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:17,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36743-0x10144fc6ca60003, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:17,372 DEBUG [Time-limited test {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-11-16T12:51:17,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36743-0x10144fc6ca60003, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:17,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35663-0x10144fc6ca60001, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:17,372 DEBUG [Time-limited test {}] master.ActiveMasterManager(353): master:35663-0x10144fc6ca60001, quorum=127.0.0.1:53208, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-11-16T12:51:17,373 DEBUG [Time-limited test {}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: killAll 2024-11-16T12:51:17,373 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 0450ab8807f5,36743,1731761465323: killAll ***** 2024-11-16T12:51:17,373 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-16T12:51:17,373 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-16T12:51:17,373 INFO [M:1;0450ab8807f5:35663 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/.lastflushedseqids 2024-11-16T12:51:17,373 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36743-0x10144fc6ca60003, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:17,374 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-16T12:51:17,377 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-16T12:51:17,377 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-16T12:51:17,377 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 301389056 }, "NonHeapMemoryUsage": { "committed": 148701184, "init": 7667712, "max": -1, "used": 146273160 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "0450ab8807f5", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 3156, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 11, "ProcessCallTime_min": 1, "ProcessCallTime_max": 8, "ProcessCallTime_mean": 3, "ProcessCallTime_25th_percentile": 2, "ProcessCallTime_median": 4, "ProcessCallTime_75th_percentile": 6, "ProcessCallTime_90th_percentile": 7, "ProcessCallTime_95th_percentile": 7, "ProcessCallTime_98th_percentile": 7, "ProcessCallTime_99th_percentile": 7, "ProcessCallTime_99.9th_percentile": 7, "ProcessCallTime_TimeRangeCount_0-1": 11, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 11, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 11, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 11, "TotalCallTime_min": 1, "TotalCallTime_max": 9, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 5 } ], "beans": [], "beans": [] } 2024-11-16T12:51:17,378 DEBUG [HMaster-EventLoopGroup-3-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0450ab8807f5,35663,1731761465058 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:17,378 WARN [Time-limited test {}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.regionserver.RegionServerAbortedException): Server 0450ab8807f5,35663,1731761465058 aborting at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:?] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:?] at org.apache.hadoop.hbase.Abortable.abort(Abortable.java:45) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster.killAll(SingleProcessHBaseCluster.java:880) ~[test-classes/:?] at org.apache.hadoop.hbase.TestZooKeeper.after(TestZooKeeper.java:107) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: Server 0450ab8807f5,35663,1731761465058 aborting at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-16T12:51:17,379 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0450ab8807f5,36743,1731761465323' ***** 2024-11-16T12:51:17,379 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: killAll 2024-11-16T12:51:17,379 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T12:51:17,379 INFO [RS:1;0450ab8807f5:36743 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T12:51:17,380 INFO [RS:1;0450ab8807f5:36743 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-16T12:51:17,380 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T12:51:17,380 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.TestZooKeeper.after(TestZooKeeper.java:110) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:51:17,380 INFO [RS:1;0450ab8807f5:36743 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-16T12:51:17,380 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:17,380 INFO [RS:1;0450ab8807f5:36743 {}] regionserver.HRegionServer(3091): Received CLOSE for 289b79361cdd89dea41ef5c11f08967b 2024-11-16T12:51:17,380 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x38112a7b to 127.0.0.1:53208 2024-11-16T12:51:17,380 INFO [regionserver/0450ab8807f5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:51:17,380 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-16T12:51:17,380 INFO [RS:1;0450ab8807f5:36743 {}] regionserver.HRegionServer(956): aborting server 0450ab8807f5,36743,1731761465323 2024-11-16T12:51:17,380 INFO [RS:1;0450ab8807f5:36743 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T12:51:17,380 INFO [RS:1;0450ab8807f5:36743 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;0450ab8807f5:36743. 2024-11-16T12:51:17,381 DEBUG [RS:1;0450ab8807f5:36743 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:51:17,380 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 289b79361cdd89dea41ef5c11f08967b, disabling compactions & flushes 2024-11-16T12:51:17,381 DEBUG [RS:1;0450ab8807f5:36743 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:17,381 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRegionServerSessionExpired_1731761468936,,1731761468956.289b79361cdd89dea41ef5c11f08967b. 2024-11-16T12:51:17,381 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionServerSessionExpired_1731761468936,,1731761468956.289b79361cdd89dea41ef5c11f08967b. 2024-11-16T12:51:17,381 INFO [RS:1;0450ab8807f5:36743 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T12:51:17,381 INFO [RS:1;0450ab8807f5:36743 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T12:51:17,381 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRegionServerSessionExpired_1731761468936,,1731761468956.289b79361cdd89dea41ef5c11f08967b. after waiting 0 ms 2024-11-16T12:51:17,381 INFO [RS:1;0450ab8807f5:36743 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T12:51:17,381 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRegionServerSessionExpired_1731761468936,,1731761468956.289b79361cdd89dea41ef5c11f08967b. 2024-11-16T12:51:17,381 INFO [RS:1;0450ab8807f5:36743 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T12:51:17,381 INFO [RS:1;0450ab8807f5:36743 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-16T12:51:17,381 DEBUG [RS:1;0450ab8807f5:36743 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 289b79361cdd89dea41ef5c11f08967b=testRegionServerSessionExpired_1731761468936,,1731761468956.289b79361cdd89dea41ef5c11f08967b.} 2024-11-16T12:51:17,381 ERROR [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1960): Memstore data size is 41 in region testRegionServerSessionExpired_1731761468936,,1731761468956.289b79361cdd89dea41ef5c11f08967b. 2024-11-16T12:51:17,381 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T12:51:17,382 DEBUG [RS:1;0450ab8807f5:36743 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 289b79361cdd89dea41ef5c11f08967b 2024-11-16T12:51:17,382 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T12:51:17,382 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRegionServerSessionExpired_1731761468936,,1731761468956.289b79361cdd89dea41ef5c11f08967b. 2024-11-16T12:51:17,382 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T12:51:17,382 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T12:51:17,382 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 289b79361cdd89dea41ef5c11f08967b: Waiting for close lock at 1731761477380Running coprocessor pre-close hooks at 1731761477380Disabling compacts and flushes for region at 1731761477380Disabling writes for close at 1731761477381 (+1 ms)Writing region close event to WAL at 1731761477381Running coprocessor post-close hooks at 1731761477381Closed at 1731761477382 (+1 ms) 2024-11-16T12:51:17,382 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T12:51:17,382 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRegionServerSessionExpired_1731761468936,,1731761468956.289b79361cdd89dea41ef5c11f08967b. 2024-11-16T12:51:17,383 ERROR [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 1723 in region hbase:meta,,1.1588230740 2024-11-16T12:51:17,383 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T12:51:17,383 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T12:51:17,383 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731761477381Running coprocessor pre-close hooks at 1731761477381Disabling compacts and flushes for region at 1731761477381Disabling writes for close at 1731761477382 (+1 ms)Writing region close event to WAL at 1731761477383 (+1 ms)Running coprocessor post-close hooks at 1731761477383Closed at 1731761477383 2024-11-16T12:51:17,384 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T12:51:17,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741841_1018 (size=128) 2024-11-16T12:51:17,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741841_1018 (size=128) 2024-11-16T12:51:17,390 INFO [M:1;0450ab8807f5:35663 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T12:51:17,390 INFO [M:1;0450ab8807f5:35663 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=true 2024-11-16T12:51:17,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741830_1006 (size=57348) 2024-11-16T12:51:17,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741830_1006 (size=57348) 2024-11-16T12:51:17,394 DEBUG [M:1;0450ab8807f5:35663 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T12:51:17,394 INFO [M:1;0450ab8807f5:35663 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:17,394 DEBUG [M:1;0450ab8807f5:35663 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:17,394 DEBUG [M:1;0450ab8807f5:35663 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T12:51:17,394 DEBUG [M:1;0450ab8807f5:35663 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:17,394 ERROR [M:1;0450ab8807f5:35663 {}] regionserver.HRegion(1960): Memstore data size is 49831 in region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:17,394 INFO [M:1;0450ab8807f5:35663 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:17,394 DEBUG [M:1;0450ab8807f5:35663 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731761477393Disabling compacts and flushes for region at 1731761477393Disabling writes for close at 1731761477394 (+1 ms)Writing region close event to WAL at 1731761477394Closed at 1731761477394 2024-11-16T12:51:17,394 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T12:51:17,394 INFO [M:1;0450ab8807f5:35663 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-16T12:51:17,395 INFO [M:1;0450ab8807f5:35663 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35663 2024-11-16T12:51:17,395 INFO [M:1;0450ab8807f5:35663 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T12:51:17,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45941-0x10144fc6ca60000, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:17,456 INFO [M:0;0450ab8807f5:45941 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T12:51:17,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45941-0x10144fc6ca60000, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:17,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35663-0x10144fc6ca60001, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:17,514 INFO [M:1;0450ab8807f5:35663 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T12:51:17,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35663-0x10144fc6ca60001, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:17,582 INFO [RS:1;0450ab8807f5:36743 {}] regionserver.HRegionServer(976): stopping server 0450ab8807f5,36743,1731761465323; all regions closed. 2024-11-16T12:51:17,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741839_1016 (size=2319) 2024-11-16T12:51:17,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741839_1016 (size=2319) 2024-11-16T12:51:17,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741834_1010 (size=607) 2024-11-16T12:51:17,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741834_1010 (size=607) 2024-11-16T12:51:17,597 DEBUG [RS:1;0450ab8807f5:36743 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:17,597 INFO [RS:1;0450ab8807f5:36743 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:51:17,597 INFO [RS:1;0450ab8807f5:36743 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T12:51:17,598 INFO [RS:1;0450ab8807f5:36743 {}] hbase.ChoreService(370): Chore service for: regionserver/0450ab8807f5:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T12:51:17,598 INFO [RS:1;0450ab8807f5:36743 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T12:51:17,598 INFO [regionserver/0450ab8807f5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T12:51:17,598 INFO [RS:1;0450ab8807f5:36743 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36743 2024-11-16T12:51:17,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36743-0x10144fc6ca60003, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0450ab8807f5,36743,1731761465323 2024-11-16T12:51:17,631 INFO [RS:1;0450ab8807f5:36743 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T12:51:17,747 INFO [RS:1;0450ab8807f5:36743 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T12:51:17,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36743-0x10144fc6ca60003, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:17,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36743-0x10144fc6ca60003, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:17,748 INFO [RS:1;0450ab8807f5:36743 {}] regionserver.HRegionServer(1031): Exiting; stopping=0450ab8807f5,36743,1731761465323; zookeeper connection closed. 2024-11-16T12:51:17,748 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@29196d1d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@29196d1d 2024-11-16T12:51:17,749 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 2 master(s) and 2 regionserver(s) complete 2024-11-16T12:51:17,750 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:17,760 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=testing utility connecting to ZooKeeper ensemble=127.0.0.1:53208 2024-11-16T12:51:17,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility0x0, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:51:17,781 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): testing utility-0x10144fc6ca60005 connected 2024-11-16T12:51:17,783 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-16T12:51:17,784 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/meta-region-server 2024-11-16T12:51:17,784 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/rs 2024-11-16T12:51:17,785 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/splitWAL 2024-11-16T12:51:17,786 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters 2024-11-16T12:51:17,787 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc 2024-11-16T12:51:17,787 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/draining 2024-11-16T12:51:17,788 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/master-maintenance 2024-11-16T12:51:17,789 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot 2024-11-16T12:51:17,790 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/hbaseid 2024-11-16T12:51:17,790 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/table 2024-11-16T12:51:17,793 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc/abort 2024-11-16T12:51:17,794 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc/acquired 2024-11-16T12:51:17,794 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc/reached 2024-11-16T12:51:17,796 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot/abort 2024-11-16T12:51:17,797 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot/acquired 2024-11-16T12:51:17,797 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot/reached 2024-11-16T12:51:17,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/reached 2024-11-16T12:51:17,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/online-snapshot 2024-11-16T12:51:17,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/acquired 2024-11-16T12:51:17,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/abort 2024-11-16T12:51:17,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/reached 2024-11-16T12:51:17,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/flush-table-proc 2024-11-16T12:51:17,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/acquired 2024-11-16T12:51:17,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/abort 2024-11-16T12:51:17,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/table 2024-11-16T12:51:17,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:17,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/hbaseid 2024-11-16T12:51:17,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot 2024-11-16T12:51:17,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master-maintenance 2024-11-16T12:51:17,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/draining 2024-11-16T12:51:17,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc 2024-11-16T12:51:17,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:17,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/splitWAL 2024-11-16T12:51:17,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs 2024-11-16T12:51:17,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:17,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T12:51:17,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase 2024-11-16T12:51:17,836 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestZooKeeper#testRegionServerSessionExpired Thread=297 (was 242) Potentially hanging thread: NIOWorkerThread-14 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1205652929) connection to localhost/127.0.0.1:44975 from jenkins.hfs.1 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: NIOWorkerThread-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NIOWorkerThread-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1205652929) connection to localhost/127.0.0.1:40751 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: NIOWorkerThread-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NIOWorkerThread-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NIOWorkerThread-15 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/0450ab8807f5:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NIOWorkerThread-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/cluster_35bff147-7ae7-a41a-0bf2-848ec5006277/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NIOWorkerThread-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NIOWorkerThread-12 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40751 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: regionserver/0450ab8807f5:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: master/0450ab8807f5:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NIOWorkerThread-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Abort regionserver monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1205652929) connection to localhost/127.0.0.1:44975 from jenkins.hfs.0 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test-SendThread(127.0.0.1:53208) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: NIOWorkerThread-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: zk-event-processor-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@63409f8b java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NIOWorkerThread-13 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:44975 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Abort regionserver monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: HMaster-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/0450ab8807f5:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:44975 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.metrics2.impl.JmxCacheBuster$JmxCacheBusterRunnable.run(JmxCacheBuster.java:101) java.base@17.0.11/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) java.base@17.0.11/java.util.concurrent.FutureTask.run(FutureTask.java:264) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NIOWorkerThread-16 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:44975 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NIOWorkerThread-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NIOWorkerThread-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=546 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=256 (was 291), ProcessCount=11 (was 11), AvailableMemoryMB=3368 (was 3685) 2024-11-16T12:51:17,846 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestZooKeeper#testMasterSessionExpired Thread=297, OpenFileDescriptor=546, MaxFileDescriptor=1048576, SystemLoadAverage=256, ProcessCount=11, AvailableMemoryMB=3367 2024-11-16T12:51:17,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741842_1019 (size=7) 2024-11-16T12:51:17,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741842_1019 (size=7) 2024-11-16T12:51:17,855 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b with version=8 2024-11-16T12:51:17,855 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/hbase-staging 2024-11-16T12:51:17,857 INFO [Time-limited test {}] client.ConnectionUtils(128): master/0450ab8807f5:0 server-side Connection retries=45 2024-11-16T12:51:17,857 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:17,857 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:17,857 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T12:51:17,857 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:17,857 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T12:51:17,857 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T12:51:17,857 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T12:51:17,858 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41747 2024-11-16T12:51:17,859 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41747 connecting to ZooKeeper ensemble=127.0.0.1:53208 2024-11-16T12:51:17,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:417470x0, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:51:17,873 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41747-0x10144fc6ca60006 connected 2024-11-16T12:51:17,877 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T12:51:17,989 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:17,991 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:17,994 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:51:17,994 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b, hbase.cluster.distributed=false 2024-11-16T12:51:17,996 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T12:51:17,997 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41747 2024-11-16T12:51:17,997 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41747 2024-11-16T12:51:17,997 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41747 2024-11-16T12:51:17,997 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41747 2024-11-16T12:51:17,998 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41747 2024-11-16T12:51:17,999 INFO [Time-limited test {}] client.ConnectionUtils(128): master/0450ab8807f5:0 server-side Connection retries=45 2024-11-16T12:51:17,999 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:17,999 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:17,999 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T12:51:17,999 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:17,999 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T12:51:17,999 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T12:51:17,999 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T12:51:18,000 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43579 2024-11-16T12:51:18,001 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43579 connecting to ZooKeeper ensemble=127.0.0.1:53208 2024-11-16T12:51:18,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:435790x0, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:51:18,023 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43579-0x10144fc6ca60007 connected 2024-11-16T12:51:18,042 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:18,044 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:18,047 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:51:18,047 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b, hbase.cluster.distributed=false 2024-11-16T12:51:18,049 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T12:51:18,049 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43579 2024-11-16T12:51:18,049 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43579 2024-11-16T12:51:18,050 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43579 2024-11-16T12:51:18,051 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43579 2024-11-16T12:51:18,051 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43579 2024-11-16T12:51:18,065 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0450ab8807f5:0 server-side Connection retries=45 2024-11-16T12:51:18,066 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:18,066 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:18,066 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T12:51:18,066 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:18,066 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T12:51:18,066 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T12:51:18,066 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T12:51:18,067 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46655 2024-11-16T12:51:18,068 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46655 connecting to ZooKeeper ensemble=127.0.0.1:53208 2024-11-16T12:51:18,069 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:18,071 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:18,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:466550x0, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:51:18,089 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:466550x0, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:51:18,089 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46655-0x10144fc6ca60008 connected 2024-11-16T12:51:18,090 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T12:51:18,090 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T12:51:18,091 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46655-0x10144fc6ca60008, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:18,092 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46655-0x10144fc6ca60008, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T12:51:18,097 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46655 2024-11-16T12:51:18,098 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46655 2024-11-16T12:51:18,098 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46655 2024-11-16T12:51:18,098 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46655 2024-11-16T12:51:18,099 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46655 2024-11-16T12:51:18,114 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0450ab8807f5:0 server-side Connection retries=45 2024-11-16T12:51:18,114 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:18,114 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:18,114 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T12:51:18,114 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:18,114 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T12:51:18,115 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T12:51:18,115 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T12:51:18,115 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46821 2024-11-16T12:51:18,117 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46821 connecting to ZooKeeper ensemble=127.0.0.1:53208 2024-11-16T12:51:18,118 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:18,120 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:18,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:468210x0, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:51:18,139 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:468210x0, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:51:18,139 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46821-0x10144fc6ca60009 connected 2024-11-16T12:51:18,140 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T12:51:18,140 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T12:51:18,141 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46821-0x10144fc6ca60009, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:18,142 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46821-0x10144fc6ca60009, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T12:51:18,146 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46821 2024-11-16T12:51:18,148 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46821 2024-11-16T12:51:18,148 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46821 2024-11-16T12:51:18,148 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46821 2024-11-16T12:51:18,149 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46821 2024-11-16T12:51:18,160 DEBUG [M:0;0450ab8807f5:41747 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0450ab8807f5:41747 2024-11-16T12:51:18,160 DEBUG [M:1;0450ab8807f5:43579 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:1;0450ab8807f5:43579 2024-11-16T12:51:18,161 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0450ab8807f5,41747,1731761477856 2024-11-16T12:51:18,161 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0450ab8807f5,43579,1731761477998 2024-11-16T12:51:18,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:18,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46821-0x10144fc6ca60009, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:18,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:18,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46655-0x10144fc6ca60008, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:18,222 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0450ab8807f5,41747,1731761477856 2024-11-16T12:51:18,223 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0450ab8807f5,43579,1731761477998 2024-11-16T12:51:18,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:18,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46821-0x10144fc6ca60009, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:18,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46655-0x10144fc6ca60008, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:18,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46821-0x10144fc6ca60009, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:18,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:18,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46655-0x10144fc6ca60008, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:18,239 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T12:51:18,240 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T12:51:18,241 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0450ab8807f5,41747,1731761477856 from backup master directory 2024-11-16T12:51:18,241 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ActiveMasterManager(296): Another master is the active master, 0450ab8807f5,41747,1731761477856; waiting to become the next active master 2024-11-16T12:51:18,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0450ab8807f5,41747,1731761477856 2024-11-16T12:51:18,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46655-0x10144fc6ca60008, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:18,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:18,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46821-0x10144fc6ca60009, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:18,247 WARN [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T12:51:18,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:18,248 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0450ab8807f5,41747,1731761477856 2024-11-16T12:51:18,255 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/hbase.id] with ID: 6497242b-2246-4e92-91d6-de5729cabcdc 2024-11-16T12:51:18,255 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/.tmp/hbase.id 2024-11-16T12:51:18,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741843_1020 (size=42) 2024-11-16T12:51:18,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741843_1020 (size=42) 2024-11-16T12:51:18,264 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/.tmp/hbase.id]:[hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/hbase.id] 2024-11-16T12:51:18,284 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:18,284 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T12:51:18,285 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-16T12:51:18,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:18,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46655-0x10144fc6ca60008, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:18,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:18,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46821-0x10144fc6ca60009, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:18,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741844_1021 (size=196) 2024-11-16T12:51:18,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741844_1021 (size=196) 2024-11-16T12:51:18,710 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T12:51:18,713 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T12:51:18,714 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:51:18,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741845_1022 (size=1189) 2024-11-16T12:51:18,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741845_1022 (size=1189) 2024-11-16T12:51:18,726 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store 2024-11-16T12:51:18,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741846_1023 (size=34) 2024-11-16T12:51:18,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741846_1023 (size=34) 2024-11-16T12:51:18,734 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:18,735 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T12:51:18,735 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:18,735 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:18,735 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T12:51:18,735 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:18,735 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:18,735 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731761478735Disabling compacts and flushes for region at 1731761478735Disabling writes for close at 1731761478735Writing region close event to WAL at 1731761478735Closed at 1731761478735 2024-11-16T12:51:18,736 WARN [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/.initializing 2024-11-16T12:51:18,736 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,41747,1731761477856 2024-11-16T12:51:18,737 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T12:51:18,740 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0450ab8807f5%2C41747%2C1731761477856, suffix=, logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,41747,1731761477856, archiveDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/oldWALs, maxLogs=10 2024-11-16T12:51:18,751 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,41747,1731761477856/0450ab8807f5%2C41747%2C1731761477856.1731761478740, exclude list is [], retry=0 2024-11-16T12:51:18,754 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36961,DS-7ef9e9a6-250e-467c-989b-a9ffcab2299e,DISK] 2024-11-16T12:51:18,754 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43257,DS-06d91055-02a4-40d7-bddb-15aee81f02ac,DISK] 2024-11-16T12:51:18,757 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,41747,1731761477856/0450ab8807f5%2C41747%2C1731761477856.1731761478740 2024-11-16T12:51:18,758 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46667:46667),(127.0.0.1/127.0.0.1:38819:38819)] 2024-11-16T12:51:18,758 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-16T12:51:18,758 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:18,758 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:18,758 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:18,761 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:18,762 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-16T12:51:18,762 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:18,763 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:18,763 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:18,765 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-16T12:51:18,765 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:18,766 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:18,766 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:18,767 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-16T12:51:18,767 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:18,768 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:18,768 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:18,769 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-16T12:51:18,770 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:18,770 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:18,770 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:18,771 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:18,772 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:18,773 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:18,773 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:18,774 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T12:51:18,776 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:18,778 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:51:18,779 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70761499, jitterRate=0.054428502917289734}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T12:51:18,780 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731761478759Initializing all the Stores at 1731761478760 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761478760Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761478760Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761478760Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761478760Cleaning up temporary data from old regions at 1731761478773 (+13 ms)Region opened successfully at 1731761478780 (+7 ms) 2024-11-16T12:51:18,780 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-16T12:51:18,784 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@397f8d35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0450ab8807f5/172.17.0.2:0 2024-11-16T12:51:18,785 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-16T12:51:18,785 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-16T12:51:18,785 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-16T12:51:18,786 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T12:51:18,786 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T12:51:18,787 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-16T12:51:18,787 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-16T12:51:18,789 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-16T12:51:18,790 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-16T12:51:18,830 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-16T12:51:18,831 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-16T12:51:18,832 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-16T12:51:18,839 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-16T12:51:18,839 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-16T12:51:18,840 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-16T12:51:18,847 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-16T12:51:18,848 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-16T12:51:18,856 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-16T12:51:18,858 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-16T12:51:18,864 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-16T12:51:18,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T12:51:18,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T12:51:18,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46655-0x10144fc6ca60008, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T12:51:18,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46821-0x10144fc6ca60009, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T12:51:18,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:18,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:18,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46655-0x10144fc6ca60008, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:18,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46821-0x10144fc6ca60009, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:18,873 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0450ab8807f5,41747,1731761477856, sessionid=0x10144fc6ca60006, setting cluster-up flag (Was=false) 2024-11-16T12:51:18,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:18,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:18,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46655-0x10144fc6ca60008, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:18,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46821-0x10144fc6ca60009, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:18,914 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-16T12:51:18,915 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0450ab8807f5,41747,1731761477856 2024-11-16T12:51:18,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:18,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:18,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46821-0x10144fc6ca60009, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:18,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46655-0x10144fc6ca60008, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:18,956 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-16T12:51:18,958 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0450ab8807f5,41747,1731761477856 2024-11-16T12:51:18,962 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-16T12:51:18,966 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T12:51:18,967 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-16T12:51:18,967 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0450ab8807f5,41747,1731761477856 Number of backup masters: 1 0450ab8807f5,43579,1731761477998 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-16T12:51:18,971 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:18,971 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:18,971 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:18,971 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:18,971 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0450ab8807f5:0, corePoolSize=10, maxPoolSize=10 2024-11-16T12:51:18,971 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:18,972 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0450ab8807f5:0, corePoolSize=2, maxPoolSize=2 2024-11-16T12:51:18,972 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:18,973 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731761508973 2024-11-16T12:51:18,973 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-16T12:51:18,973 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-16T12:51:18,973 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-16T12:51:18,973 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-16T12:51:18,973 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-16T12:51:18,973 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-16T12:51:18,974 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:18,974 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T12:51:18,974 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-16T12:51:18,974 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-16T12:51:18,974 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-16T12:51:18,974 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-16T12:51:18,975 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-16T12:51:18,975 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-16T12:51:18,975 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.large.0-1731761478975,5,FailOnTimeoutGroup] 2024-11-16T12:51:18,975 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.small.0-1731761478975,5,FailOnTimeoutGroup] 2024-11-16T12:51:18,976 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:18,976 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-16T12:51:18,976 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:18,976 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:18,976 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:18,976 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-16T12:51:18,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741848_1025 (size=1321) 2024-11-16T12:51:18,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741848_1025 (size=1321) 2024-11-16T12:51:18,986 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-16T12:51:18,986 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:18,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741849_1026 (size=32) 2024-11-16T12:51:18,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741849_1026 (size=32) 2024-11-16T12:51:18,994 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:18,996 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T12:51:18,998 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T12:51:18,998 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:18,999 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:18,999 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T12:51:19,001 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T12:51:19,001 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:19,002 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:19,002 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T12:51:19,003 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T12:51:19,003 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:19,004 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:19,004 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T12:51:19,006 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T12:51:19,006 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:19,007 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:19,007 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T12:51:19,008 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740 2024-11-16T12:51:19,008 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740 2024-11-16T12:51:19,010 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T12:51:19,010 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T12:51:19,011 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T12:51:19,012 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T12:51:19,015 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:51:19,015 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63426929, jitterRate=-0.05486510694026947}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T12:51:19,016 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731761478995Initializing all the Stores at 1731761478996 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761478996Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761478996Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761478996Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761478996Cleaning up temporary data from old regions at 1731761479010 (+14 ms)Region opened successfully at 1731761479016 (+6 ms) 2024-11-16T12:51:19,016 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T12:51:19,016 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T12:51:19,016 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T12:51:19,016 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T12:51:19,016 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T12:51:19,017 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T12:51:19,017 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731761479016Disabling compacts and flushes for region at 1731761479016Disabling writes for close at 1731761479016Writing region close event to WAL at 1731761479017 (+1 ms)Closed at 1731761479017 2024-11-16T12:51:19,018 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T12:51:19,018 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-16T12:51:19,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T12:51:19,020 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T12:51:19,022 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-16T12:51:19,052 INFO [RS:1;0450ab8807f5:46821 {}] regionserver.HRegionServer(746): ClusterId : 6497242b-2246-4e92-91d6-de5729cabcdc 2024-11-16T12:51:19,052 INFO [RS:0;0450ab8807f5:46655 {}] regionserver.HRegionServer(746): ClusterId : 6497242b-2246-4e92-91d6-de5729cabcdc 2024-11-16T12:51:19,053 DEBUG [RS:1;0450ab8807f5:46821 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T12:51:19,053 DEBUG [RS:0;0450ab8807f5:46655 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T12:51:19,073 DEBUG [RS:1;0450ab8807f5:46821 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T12:51:19,073 DEBUG [RS:0;0450ab8807f5:46655 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T12:51:19,073 DEBUG [RS:1;0450ab8807f5:46821 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T12:51:19,073 DEBUG [RS:0;0450ab8807f5:46655 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T12:51:19,081 DEBUG [RS:1;0450ab8807f5:46821 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T12:51:19,082 DEBUG [RS:0;0450ab8807f5:46655 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T12:51:19,082 DEBUG [RS:1;0450ab8807f5:46821 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@296e89c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0450ab8807f5/172.17.0.2:0 2024-11-16T12:51:19,082 DEBUG [RS:0;0450ab8807f5:46655 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21b857c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0450ab8807f5/172.17.0.2:0 2024-11-16T12:51:19,098 DEBUG [RS:0;0450ab8807f5:46655 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0450ab8807f5:46655 2024-11-16T12:51:19,098 DEBUG [RS:1;0450ab8807f5:46821 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;0450ab8807f5:46821 2024-11-16T12:51:19,098 INFO [RS:0;0450ab8807f5:46655 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T12:51:19,098 INFO [RS:1;0450ab8807f5:46821 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T12:51:19,098 INFO [RS:0;0450ab8807f5:46655 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T12:51:19,098 INFO [RS:1;0450ab8807f5:46821 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T12:51:19,098 DEBUG [RS:0;0450ab8807f5:46655 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T12:51:19,098 DEBUG [RS:1;0450ab8807f5:46821 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T12:51:19,099 INFO [RS:1;0450ab8807f5:46821 {}] regionserver.HRegionServer(2659): reportForDuty to master=0450ab8807f5,41747,1731761477856 with port=46821, startcode=1731761478114 2024-11-16T12:51:19,099 INFO [RS:0;0450ab8807f5:46655 {}] regionserver.HRegionServer(2659): reportForDuty to master=0450ab8807f5,41747,1731761477856 with port=46655, startcode=1731761478065 2024-11-16T12:51:19,099 DEBUG [RS:0;0450ab8807f5:46655 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T12:51:19,099 DEBUG [RS:1;0450ab8807f5:46821 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T12:51:19,101 INFO [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40585, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T12:51:19,101 INFO [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41773, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T12:51:19,101 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41747 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0450ab8807f5,46655,1731761478065 2024-11-16T12:51:19,102 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41747 {}] master.ServerManager(517): Registering regionserver=0450ab8807f5,46655,1731761478065 2024-11-16T12:51:19,103 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41747 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0450ab8807f5,46821,1731761478114 2024-11-16T12:51:19,103 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41747 {}] master.ServerManager(517): Registering regionserver=0450ab8807f5,46821,1731761478114 2024-11-16T12:51:19,104 DEBUG [RS:0;0450ab8807f5:46655 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:19,104 DEBUG [RS:0;0450ab8807f5:46655 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44975 2024-11-16T12:51:19,104 DEBUG [RS:0;0450ab8807f5:46655 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T12:51:19,105 DEBUG [RS:1;0450ab8807f5:46821 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:19,105 DEBUG [RS:1;0450ab8807f5:46821 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44975 2024-11-16T12:51:19,105 DEBUG [RS:1;0450ab8807f5:46821 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T12:51:19,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T12:51:19,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T12:51:19,122 DEBUG [RS:0;0450ab8807f5:46655 {}] zookeeper.ZKUtil(111): regionserver:46655-0x10144fc6ca60008, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0450ab8807f5,46655,1731761478065 2024-11-16T12:51:19,122 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0450ab8807f5,46821,1731761478114] 2024-11-16T12:51:19,122 WARN [RS:0;0450ab8807f5:46655 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T12:51:19,122 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0450ab8807f5,46655,1731761478065] 2024-11-16T12:51:19,122 INFO [RS:0;0450ab8807f5:46655 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:51:19,122 DEBUG [RS:1;0450ab8807f5:46821 {}] zookeeper.ZKUtil(111): regionserver:46821-0x10144fc6ca60009, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0450ab8807f5,46821,1731761478114 2024-11-16T12:51:19,123 WARN [RS:1;0450ab8807f5:46821 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T12:51:19,123 DEBUG [RS:0;0450ab8807f5:46655 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,46655,1731761478065 2024-11-16T12:51:19,123 INFO [RS:1;0450ab8807f5:46821 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:51:19,123 DEBUG [RS:1;0450ab8807f5:46821 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,46821,1731761478114 2024-11-16T12:51:19,128 INFO [RS:0;0450ab8807f5:46655 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T12:51:19,128 INFO [RS:1;0450ab8807f5:46821 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T12:51:19,130 INFO [RS:1;0450ab8807f5:46821 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T12:51:19,130 INFO [RS:1;0450ab8807f5:46821 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T12:51:19,130 INFO [RS:1;0450ab8807f5:46821 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:19,132 INFO [RS:1;0450ab8807f5:46821 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T12:51:19,132 INFO [RS:0;0450ab8807f5:46655 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T12:51:19,132 INFO [RS:0;0450ab8807f5:46655 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T12:51:19,133 INFO [RS:0;0450ab8807f5:46655 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:19,133 INFO [RS:0;0450ab8807f5:46655 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T12:51:19,133 INFO [RS:1;0450ab8807f5:46821 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T12:51:19,133 INFO [RS:1;0450ab8807f5:46821 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:19,133 DEBUG [RS:1;0450ab8807f5:46821 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:19,134 DEBUG [RS:1;0450ab8807f5:46821 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:19,134 DEBUG [RS:1;0450ab8807f5:46821 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:19,134 DEBUG [RS:1;0450ab8807f5:46821 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:19,134 DEBUG [RS:1;0450ab8807f5:46821 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:19,134 DEBUG [RS:1;0450ab8807f5:46821 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0, corePoolSize=2, maxPoolSize=2 2024-11-16T12:51:19,134 DEBUG [RS:1;0450ab8807f5:46821 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:19,134 INFO [RS:0;0450ab8807f5:46655 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T12:51:19,134 DEBUG [RS:1;0450ab8807f5:46821 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:19,134 INFO [RS:0;0450ab8807f5:46655 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:19,134 DEBUG [RS:1;0450ab8807f5:46821 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:19,134 DEBUG [RS:1;0450ab8807f5:46821 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:19,134 DEBUG [RS:0;0450ab8807f5:46655 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:19,134 DEBUG [RS:1;0450ab8807f5:46821 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:19,134 DEBUG [RS:0;0450ab8807f5:46655 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:19,135 DEBUG [RS:0;0450ab8807f5:46655 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:19,135 DEBUG [RS:1;0450ab8807f5:46821 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:19,135 DEBUG [RS:1;0450ab8807f5:46821 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:51:19,135 DEBUG [RS:0;0450ab8807f5:46655 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:19,135 DEBUG [RS:0;0450ab8807f5:46655 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:19,135 DEBUG [RS:1;0450ab8807f5:46821 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:51:19,135 DEBUG [RS:0;0450ab8807f5:46655 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0, corePoolSize=2, maxPoolSize=2 2024-11-16T12:51:19,135 DEBUG [RS:0;0450ab8807f5:46655 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:19,135 DEBUG [RS:0;0450ab8807f5:46655 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:19,135 DEBUG [RS:0;0450ab8807f5:46655 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:19,135 INFO [RS:1;0450ab8807f5:46821 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:19,135 DEBUG [RS:0;0450ab8807f5:46655 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:19,135 INFO [RS:1;0450ab8807f5:46821 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:19,135 DEBUG [RS:0;0450ab8807f5:46655 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:19,135 INFO [RS:1;0450ab8807f5:46821 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:19,135 INFO [RS:1;0450ab8807f5:46821 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:19,135 DEBUG [RS:0;0450ab8807f5:46655 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:19,135 INFO [RS:1;0450ab8807f5:46821 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:19,135 DEBUG [RS:0;0450ab8807f5:46655 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:51:19,135 INFO [RS:1;0450ab8807f5:46821 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,46821,1731761478114-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T12:51:19,136 DEBUG [RS:0;0450ab8807f5:46655 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:51:19,136 INFO [RS:0;0450ab8807f5:46655 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:19,136 INFO [RS:0;0450ab8807f5:46655 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:19,136 INFO [RS:0;0450ab8807f5:46655 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:19,136 INFO [RS:0;0450ab8807f5:46655 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:19,136 INFO [RS:0;0450ab8807f5:46655 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:19,136 INFO [RS:0;0450ab8807f5:46655 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,46655,1731761478065-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T12:51:19,149 INFO [RS:1;0450ab8807f5:46821 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T12:51:19,149 INFO [RS:0;0450ab8807f5:46655 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T12:51:19,149 INFO [RS:1;0450ab8807f5:46821 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,46821,1731761478114-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:19,149 INFO [RS:0;0450ab8807f5:46655 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,46655,1731761478065-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:19,149 INFO [RS:0;0450ab8807f5:46655 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:19,149 INFO [RS:1;0450ab8807f5:46821 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:19,149 INFO [RS:0;0450ab8807f5:46655 {}] regionserver.Replication(171): 0450ab8807f5,46655,1731761478065 started 2024-11-16T12:51:19,149 INFO [RS:1;0450ab8807f5:46821 {}] regionserver.Replication(171): 0450ab8807f5,46821,1731761478114 started 2024-11-16T12:51:19,162 INFO [RS:0;0450ab8807f5:46655 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:19,162 INFO [RS:1;0450ab8807f5:46821 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:19,162 INFO [RS:0;0450ab8807f5:46655 {}] regionserver.HRegionServer(1482): Serving as 0450ab8807f5,46655,1731761478065, RpcServer on 0450ab8807f5/172.17.0.2:46655, sessionid=0x10144fc6ca60008 2024-11-16T12:51:19,162 INFO [RS:1;0450ab8807f5:46821 {}] regionserver.HRegionServer(1482): Serving as 0450ab8807f5,46821,1731761478114, RpcServer on 0450ab8807f5/172.17.0.2:46821, sessionid=0x10144fc6ca60009 2024-11-16T12:51:19,162 DEBUG [RS:0;0450ab8807f5:46655 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T12:51:19,162 DEBUG [RS:1;0450ab8807f5:46821 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T12:51:19,162 DEBUG [RS:0;0450ab8807f5:46655 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0450ab8807f5,46655,1731761478065 2024-11-16T12:51:19,162 DEBUG [RS:1;0450ab8807f5:46821 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0450ab8807f5,46821,1731761478114 2024-11-16T12:51:19,162 DEBUG [RS:0;0450ab8807f5:46655 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,46655,1731761478065' 2024-11-16T12:51:19,162 DEBUG [RS:1;0450ab8807f5:46821 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,46821,1731761478114' 2024-11-16T12:51:19,162 DEBUG [RS:0;0450ab8807f5:46655 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T12:51:19,162 DEBUG [RS:1;0450ab8807f5:46821 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T12:51:19,163 DEBUG [RS:0;0450ab8807f5:46655 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T12:51:19,163 DEBUG [RS:1;0450ab8807f5:46821 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T12:51:19,164 DEBUG [RS:0;0450ab8807f5:46655 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T12:51:19,164 DEBUG [RS:1;0450ab8807f5:46821 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T12:51:19,164 DEBUG [RS:0;0450ab8807f5:46655 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T12:51:19,164 DEBUG [RS:1;0450ab8807f5:46821 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T12:51:19,164 DEBUG [RS:0;0450ab8807f5:46655 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0450ab8807f5,46655,1731761478065 2024-11-16T12:51:19,164 DEBUG [RS:1;0450ab8807f5:46821 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0450ab8807f5,46821,1731761478114 2024-11-16T12:51:19,164 DEBUG [RS:0;0450ab8807f5:46655 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,46655,1731761478065' 2024-11-16T12:51:19,164 DEBUG [RS:1;0450ab8807f5:46821 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,46821,1731761478114' 2024-11-16T12:51:19,164 DEBUG [RS:0;0450ab8807f5:46655 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T12:51:19,164 DEBUG [RS:1;0450ab8807f5:46821 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T12:51:19,164 DEBUG [RS:0;0450ab8807f5:46655 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T12:51:19,164 DEBUG [RS:1;0450ab8807f5:46821 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T12:51:19,164 DEBUG [RS:0;0450ab8807f5:46655 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T12:51:19,164 DEBUG [RS:1;0450ab8807f5:46821 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T12:51:19,164 INFO [RS:0;0450ab8807f5:46655 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T12:51:19,164 INFO [RS:1;0450ab8807f5:46821 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T12:51:19,164 INFO [RS:0;0450ab8807f5:46655 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T12:51:19,164 INFO [RS:1;0450ab8807f5:46821 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T12:51:19,172 WARN [0450ab8807f5:41747 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-16T12:51:19,266 INFO [RS:1;0450ab8807f5:46821 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T12:51:19,266 INFO [RS:0;0450ab8807f5:46655 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T12:51:19,272 INFO [RS:1;0450ab8807f5:46821 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0450ab8807f5%2C46821%2C1731761478114, suffix=, logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,46821,1731761478114, archiveDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/oldWALs, maxLogs=32 2024-11-16T12:51:19,272 INFO [RS:0;0450ab8807f5:46655 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0450ab8807f5%2C46655%2C1731761478065, suffix=, logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,46655,1731761478065, archiveDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/oldWALs, maxLogs=32 2024-11-16T12:51:19,288 DEBUG [RS:0;0450ab8807f5:46655 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,46655,1731761478065/0450ab8807f5%2C46655%2C1731761478065.1731761479276, exclude list is [], retry=0 2024-11-16T12:51:19,288 DEBUG [RS:1;0450ab8807f5:46821 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,46821,1731761478114/0450ab8807f5%2C46821%2C1731761478114.1731761479276, exclude list is [], retry=0 2024-11-16T12:51:19,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36961,DS-7ef9e9a6-250e-467c-989b-a9ffcab2299e,DISK] 2024-11-16T12:51:19,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43257,DS-06d91055-02a4-40d7-bddb-15aee81f02ac,DISK] 2024-11-16T12:51:19,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36961,DS-7ef9e9a6-250e-467c-989b-a9ffcab2299e,DISK] 2024-11-16T12:51:19,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43257,DS-06d91055-02a4-40d7-bddb-15aee81f02ac,DISK] 2024-11-16T12:51:19,294 INFO [RS:0;0450ab8807f5:46655 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,46655,1731761478065/0450ab8807f5%2C46655%2C1731761478065.1731761479276 2024-11-16T12:51:19,295 INFO [RS:1;0450ab8807f5:46821 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,46821,1731761478114/0450ab8807f5%2C46821%2C1731761478114.1731761479276 2024-11-16T12:51:19,296 DEBUG [RS:0;0450ab8807f5:46655 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46667:46667),(127.0.0.1/127.0.0.1:38819:38819)] 2024-11-16T12:51:19,297 DEBUG [RS:1;0450ab8807f5:46821 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46667:46667),(127.0.0.1/127.0.0.1:38819:38819)] 2024-11-16T12:51:19,422 DEBUG [0450ab8807f5:41747 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-16T12:51:19,423 DEBUG [0450ab8807f5:41747 {}] balancer.BalancerClusterState(204): Hosts are {0450ab8807f5=0} racks are {/default-rack=0} 2024-11-16T12:51:19,425 DEBUG [0450ab8807f5:41747 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T12:51:19,425 DEBUG [0450ab8807f5:41747 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T12:51:19,425 DEBUG [0450ab8807f5:41747 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T12:51:19,425 DEBUG [0450ab8807f5:41747 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T12:51:19,425 INFO [0450ab8807f5:41747 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T12:51:19,425 INFO [0450ab8807f5:41747 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T12:51:19,425 DEBUG [0450ab8807f5:41747 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T12:51:19,426 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0450ab8807f5,46821,1731761478114 2024-11-16T12:51:19,427 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0450ab8807f5,46821,1731761478114, state=OPENING 2024-11-16T12:51:19,455 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-16T12:51:19,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46821-0x10144fc6ca60009, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:19,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:19,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:19,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46655-0x10144fc6ca60008, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:19,465 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T12:51:19,465 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:19,465 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:19,465 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0450ab8807f5,46821,1731761478114}] 2024-11-16T12:51:19,465 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:19,465 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:19,621 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T12:51:19,625 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58671, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T12:51:19,632 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T12:51:19,633 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:51:19,633 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T12:51:19,635 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0450ab8807f5%2C46821%2C1731761478114.meta, suffix=.meta, logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,46821,1731761478114, archiveDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/oldWALs, maxLogs=32 2024-11-16T12:51:19,648 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,46821,1731761478114/0450ab8807f5%2C46821%2C1731761478114.meta.1731761479636.meta, exclude list is [], retry=0 2024-11-16T12:51:19,652 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43257,DS-06d91055-02a4-40d7-bddb-15aee81f02ac,DISK] 2024-11-16T12:51:19,652 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36961,DS-7ef9e9a6-250e-467c-989b-a9ffcab2299e,DISK] 2024-11-16T12:51:19,654 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,46821,1731761478114/0450ab8807f5%2C46821%2C1731761478114.meta.1731761479636.meta 2024-11-16T12:51:19,654 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46667:46667),(127.0.0.1/127.0.0.1:38819:38819)] 2024-11-16T12:51:19,655 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T12:51:19,655 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T12:51:19,655 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T12:51:19,655 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T12:51:19,655 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T12:51:19,655 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:19,656 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T12:51:19,656 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T12:51:19,658 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T12:51:19,659 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T12:51:19,659 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:19,660 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:19,660 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T12:51:19,662 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T12:51:19,662 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:19,662 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:19,663 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T12:51:19,664 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T12:51:19,664 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:19,665 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:19,665 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T12:51:19,666 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T12:51:19,666 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:19,667 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:19,667 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T12:51:19,668 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740 2024-11-16T12:51:19,669 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740 2024-11-16T12:51:19,671 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T12:51:19,671 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T12:51:19,672 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T12:51:19,674 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T12:51:19,675 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62966516, jitterRate=-0.06172579526901245}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T12:51:19,675 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T12:51:19,676 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731761479656Writing region info on filesystem at 1731761479656Initializing all the Stores at 1731761479657 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761479657Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761479658 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761479658Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761479658Cleaning up temporary data from old regions at 1731761479671 (+13 ms)Running coprocessor post-open hooks at 1731761479675 (+4 ms)Region opened successfully at 1731761479676 (+1 ms) 2024-11-16T12:51:19,677 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731761479621 2024-11-16T12:51:19,680 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T12:51:19,680 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T12:51:19,681 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0450ab8807f5,46821,1731761478114 2024-11-16T12:51:19,682 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0450ab8807f5,46821,1731761478114, state=OPEN 2024-11-16T12:51:19,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46655-0x10144fc6ca60008, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:19,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:19,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:19,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46821-0x10144fc6ca60009, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:19,714 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:19,714 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:19,714 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:19,714 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0450ab8807f5,46821,1731761478114 2024-11-16T12:51:19,714 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:19,718 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-16T12:51:19,718 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0450ab8807f5,46821,1731761478114 in 249 msec 2024-11-16T12:51:19,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-16T12:51:19,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 699 msec 2024-11-16T12:51:19,723 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T12:51:19,723 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-16T12:51:19,725 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T12:51:19,725 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,46821,1731761478114, seqNum=-1] 2024-11-16T12:51:19,725 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T12:51:19,727 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56611, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T12:51:19,734 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 769 msec 2024-11-16T12:51:19,734 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731761479734, completionTime=-1 2024-11-16T12:51:19,735 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-16T12:51:19,735 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T12:51:19,737 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T12:51:19,738 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731761539737 2024-11-16T12:51:19,738 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731761599738 2024-11-16T12:51:19,738 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-16T12:51:19,738 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-16T12:51:19,739 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,41747,1731761477856-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:19,739 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,41747,1731761477856-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:19,739 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,41747,1731761477856-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:19,739 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0450ab8807f5:41747, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:19,739 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:19,740 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:19,741 DEBUG [master/0450ab8807f5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-16T12:51:19,744 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.496sec 2024-11-16T12:51:19,744 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-16T12:51:19,744 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-16T12:51:19,744 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-16T12:51:19,744 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-16T12:51:19,744 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-16T12:51:19,744 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,41747,1731761477856-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T12:51:19,745 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,41747,1731761477856-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-16T12:51:19,747 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-16T12:51:19,747 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-16T12:51:19,747 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,41747,1731761477856-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:19,753 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0b9b1968 to 127.0.0.1:53208 with session timeout=1000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@68207fb7 2024-11-16T12:51:19,765 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58bdafe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T12:51:19,765 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T12:51:19,766 DEBUG [ReadOnlyZKClient-127.0.0.1:53208@0x0b9b1968 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,46821,1731761478114, seqNum=-1] 2024-11-16T12:51:19,767 DEBUG [ReadOnlyZKClient-127.0.0.1:53208@0x0b9b1968 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T12:51:19,769 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51524, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T12:51:19,771 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0450ab8807f5,41747,1731761477856 2024-11-16T12:51:19,772 INFO [Time-limited test {}] hbase.TestZooKeeper(127): Starting testMasterSessionExpired 2024-11-16T12:51:19,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:19,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46821-0x10144fc6ca60009, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:19,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46655-0x10144fc6ca60008, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:19,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:19,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46655-0x10144fc6ca60008, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:19,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46821-0x10144fc6ca60009, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:19,781 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:19,781 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46821-0x10144fc6ca60009, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:19,781 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46655-0x10144fc6ca60008, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:19,781 DEBUG [zk-event-processor-pool-0 {}] master.ActiveMasterManager(204): No master available. Notifying waiting threads 2024-11-16T12:51:19,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46821-0x10144fc6ca60009, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:19,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46655-0x10144fc6ca60008, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:19,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:19,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46821-0x10144fc6ca60009, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:19,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46655-0x10144fc6ca60008, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:19,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:19,822 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T12:51:19,822 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T12:51:19,823 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0450ab8807f5,43579,1731761477998 from backup master directory 2024-11-16T12:51:19,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0450ab8807f5,43579,1731761477998 2024-11-16T12:51:19,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46821-0x10144fc6ca60009, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:19,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46655-0x10144fc6ca60008, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:19,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:19,839 WARN [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T12:51:19,839 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0450ab8807f5,43579,1731761477998 2024-11-16T12:51:19,854 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:19,854 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T12:51:19,865 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=1) cost 10ms. 2024-11-16T12:51:19,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Disconnected, path=null 2024-11-16T12:51:19,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Disconnected, path=null 2024-11-16T12:51:19,874 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(640): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Received Disconnected from ZooKeeper, ignoring 2024-11-16T12:51:19,875 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(640): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Received Disconnected from ZooKeeper, ignoring 2024-11-16T12:51:19,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741853_1030 (size=196) 2024-11-16T12:51:19,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741853_1030 (size=196) 2024-11-16T12:51:19,882 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T12:51:19,882 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2501): ZK Closed Session 0x10144fc6ca60006 2024-11-16T12:51:19,883 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T12:51:19,883 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:51:19,894 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(342): old store file tracker DEFAULT is the same with new store file tracker, skip migration 2024-11-16T12:51:19,897 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(316): Renamed hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,41747,1731761477856 to hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,41747,1731761477856-dead as it is dead 2024-11-16T12:51:19,898 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,41747,1731761477856-dead/0450ab8807f5%2C41747%2C1731761477856.1731761478740 2024-11-16T12:51:19,899 WARN [IPC Server handler 0 on default port 44975 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,41747,1731761477856-dead/0450ab8807f5%2C41747%2C1731761477856.1731761478740 has not been closed. Lease recovery is in progress. RecoveryId = 1031 for block blk_1073741847_1024 2024-11-16T12:51:19,899 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,41747,1731761477856-dead/0450ab8807f5%2C41747%2C1731761477856.1731761478740 after 1ms 2024-11-16T12:51:21,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Expired, path=null 2024-11-16T12:51:21,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Expired, path=null 2024-11-16T12:51:21,206 ERROR [zk-event-processor-pool-0 {}] master.HMaster(3218): Master server abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-16T12:51:21,206 ERROR [zk-event-processor-pool-0 {}] master.HMaster(3223): ***** ABORTING master 0450ab8807f5,41747,1731761477856: master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase master:41747-0x10144fc6ca60006 received expired from ZooKeeper, aborting ***** org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired at org.apache.hadoop.hbase.zookeeper.ZKWatcher.connectionEvent(ZKWatcher.java:652) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.lambda$processEvent$0(ZKWatcher.java:563) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.processEvent(ZKWatcher.java:559) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.lambda$process$1(ZKWatcher.java:613) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:21,207 INFO [zk-event-processor-pool-0 {}] master.HMaster(3321): ***** STOPPING master '0450ab8807f5,41747,1731761477856' ***** 2024-11-16T12:51:21,207 INFO [zk-event-processor-pool-0 {}] master.HMaster(3323): STOPPED: Stopped by zk-event-processor-pool-0 2024-11-16T12:51:21,207 INFO [M:0;0450ab8807f5:41747 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T12:51:21,207 INFO [M:0;0450ab8807f5:41747 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;0450ab8807f5:41747. 2024-11-16T12:51:21,208 DEBUG [M:0;0450ab8807f5:41747 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:51:21,208 DEBUG [M:0;0450ab8807f5:41747 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:21,207 WARN [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(351): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/master org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /hbase/master at org.apache.zookeeper.KeeperException.create(KeeperException.java:134) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1972) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:368) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:356) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.getData(ZKUtil.java:343) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.MasterAddressTracker.getMasterAddress(MasterAddressTracker.java:171) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.ActiveMasterManager.stop(ActiveMasterManager.java:342) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.stop(HMaster.java:3327) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.stopMaster(HMaster.java:3315) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.abort(HMaster.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.connectionEvent(ZKWatcher.java:652) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.lambda$processEvent$0(ZKWatcher.java:563) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.processEvent(ZKWatcher.java:559) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.lambda$process$1(ZKWatcher.java:613) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:21,208 INFO [M:0;0450ab8807f5:41747 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T12:51:21,208 DEBUG [M:0;0450ab8807f5:41747 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T12:51:21,209 DEBUG [M:0;0450ab8807f5:41747 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T12:51:21,209 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T12:51:21,209 DEBUG [master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.large.0-1731761478975 {}] cleaner.HFileCleaner(306): Exit Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.large.0-1731761478975,5,FailOnTimeoutGroup] 2024-11-16T12:51:21,209 ERROR [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(711): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Received unexpected KeeperException, re-throwing exception org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /hbase/master at org.apache.zookeeper.KeeperException.create(KeeperException.java:134) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1972) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:368) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:356) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.getData(ZKUtil.java:343) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.MasterAddressTracker.getMasterAddress(MasterAddressTracker.java:171) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.ActiveMasterManager.stop(ActiveMasterManager.java:342) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.stop(HMaster.java:3327) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.stopMaster(HMaster.java:3315) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.abort(HMaster.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.connectionEvent(ZKWatcher.java:652) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.lambda$processEvent$0(ZKWatcher.java:563) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.processEvent(ZKWatcher.java:559) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.lambda$process$1(ZKWatcher.java:613) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:21,209 DEBUG [zk-event-processor-pool-0 {}] master.ActiveMasterManager(353): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = Session expired for /hbase/master 2024-11-16T12:51:21,209 DEBUG [master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.small.0-1731761478975 {}] cleaner.HFileCleaner(306): Exit Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.small.0-1731761478975,5,FailOnTimeoutGroup] 2024-11-16T12:51:21,210 DEBUG [zk-event-processor-pool-0 {}] master.HMaster(3213): Abort called but aborted=true, stopped=true 2024-11-16T12:51:21,210 INFO [M:0;0450ab8807f5:41747 {}] hbase.ChoreService(370): Chore service for: master/0450ab8807f5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T12:51:21,210 INFO [M:0;0450ab8807f5:41747 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T12:51:21,210 DEBUG [M:0;0450ab8807f5:41747 {}] master.HMaster(1795): Stopping service threads 2024-11-16T12:51:21,210 INFO [M:0;0450ab8807f5:41747 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T12:51:21,210 INFO [M:0;0450ab8807f5:41747 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T12:51:21,211 INFO [M:0;0450ab8807f5:41747 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=true 2024-11-16T12:51:21,211 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T12:51:21,211 WARN [M:0;0450ab8807f5:41747 {}] zookeeper.ZKUtil(351): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/master org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /hbase/master at org.apache.zookeeper.KeeperException.create(KeeperException.java:134) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1972) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:368) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:356) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.getData(ZKUtil.java:343) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.MasterAddressTracker.getMasterAddress(MasterAddressTracker.java:171) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.ActiveMasterManager.stop(ActiveMasterManager.java:342) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.stopServiceThreads(HMaster.java:1809) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:631) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:21,211 ERROR [M:0;0450ab8807f5:41747 {}] zookeeper.ZKWatcher(711): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Received unexpected KeeperException, re-throwing exception org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /hbase/master at org.apache.zookeeper.KeeperException.create(KeeperException.java:134) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1972) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:368) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:356) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.getData(ZKUtil.java:343) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.MasterAddressTracker.getMasterAddress(MasterAddressTracker.java:171) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.ActiveMasterManager.stop(ActiveMasterManager.java:342) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.stopServiceThreads(HMaster.java:1809) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:631) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:21,211 DEBUG [M:0;0450ab8807f5:41747 {}] master.ActiveMasterManager(353): master:41747-0x10144fc6ca60006, quorum=127.0.0.1:53208, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = Session expired for /hbase/master 2024-11-16T12:51:21,212 INFO [M:0;0450ab8807f5:41747 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/.lastflushedseqids 2024-11-16T12:51:21,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741854_1032 (size=99) 2024-11-16T12:51:21,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741854_1032 (size=99) 2024-11-16T12:51:21,220 INFO [M:0;0450ab8807f5:41747 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T12:51:21,220 INFO [M:0;0450ab8807f5:41747 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=true 2024-11-16T12:51:21,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741847_1024 (size=9664) 2024-11-16T12:51:21,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741847_1024 (size=9664) 2024-11-16T12:51:21,223 WARN [Close-WAL-Writer-0 {}] wal.AsyncProtobufLogWriter(165): normal close failed, try recover java.io.IOException: Commit or complete block blk_1073741847_1024, whereas it is under recovery. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.commitOrCompleteLastBlock(BlockManager.java:1211) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.commitOrCompleteLastBlock(FSNamesystem.java:3890) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:731) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:163) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: Commit or complete block blk_1073741847_1024, whereas it is under recovery. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.commitOrCompleteLastBlock(BlockManager.java:1211) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.commitOrCompleteLastBlock(FSNamesystem.java:3890) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:731) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 6 more 2024-11-16T12:51:21,224 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,41747,1731761477856/0450ab8807f5%2C41747%2C1731761477856.1731761478740 2024-11-16T12:51:21,224 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,41747,1731761477856/0450ab8807f5%2C41747%2C1731761477856.1731761478740 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.recoverLease(FSNamesystem.java:2869) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.recoverLease(NameNodeRpcServer.java:872) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.recoverLease(ClientNamenodeProtocolServerSideTranslatorPB.java:834) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:951) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.recoverAndClose(FanOutOneBlockAsyncDFSOutput.java:605) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:166) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,41747,1731761477856/0450ab8807f5%2C41747%2C1731761477856.1731761478740 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.recoverLease(FSNamesystem.java:2869) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.recoverLease(NameNodeRpcServer.java:872) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.recoverLease(ClientNamenodeProtocolServerSideTranslatorPB.java:834) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.recoverLease(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$recoverLease$24(ClientNamenodeProtocolTranslatorPB.java:685) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.recoverLease(ClientNamenodeProtocolTranslatorPB.java:685) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:949) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 17 more 2024-11-16T12:51:21,225 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,41747,1731761477856/0450ab8807f5%2C41747%2C1731761477856.1731761478740 2024-11-16T12:51:21,226 ERROR [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2033): Unable to recover lease after several attempts. Give up. java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,41747,1731761477856/0450ab8807f5%2C41747%2C1731761477856.1731761478740 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.recoverLease(FSNamesystem.java:2869) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.recoverLease(NameNodeRpcServer.java:872) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.recoverLease(ClientNamenodeProtocolServerSideTranslatorPB.java:834) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:951) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,41747,1731761477856/0450ab8807f5%2C41747%2C1731761477856.1731761478740 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.recoverLease(FSNamesystem.java:2869) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.recoverLease(NameNodeRpcServer.java:872) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.recoverLease(ClientNamenodeProtocolServerSideTranslatorPB.java:834) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.recoverLease(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$recoverLease$24(ClientNamenodeProtocolTranslatorPB.java:685) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.recoverLease(ClientNamenodeProtocolTranslatorPB.java:685) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:949) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 16 more 2024-11-16T12:51:21,226 DEBUG [M:0;0450ab8807f5:41747 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T12:51:21,226 INFO [M:0;0450ab8807f5:41747 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:21,227 DEBUG [M:0;0450ab8807f5:41747 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:21,227 DEBUG [M:0;0450ab8807f5:41747 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T12:51:21,227 DEBUG [M:0;0450ab8807f5:41747 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:21,227 ERROR [M:0;0450ab8807f5:41747 {}] regionserver.HRegion(1960): Memstore data size is 7867 in region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:21,227 INFO [M:0;0450ab8807f5:41747 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:21,227 DEBUG [M:0;0450ab8807f5:41747 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731761481226Disabling compacts and flushes for region at 1731761481226Disabling writes for close at 1731761481227 (+1 ms)Writing region close event to WAL at 1731761481227Closed at 1731761481227 2024-11-16T12:51:21,227 INFO [M:0;0450ab8807f5:41747 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-16T12:51:21,227 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T12:51:21,227 INFO [M:0;0450ab8807f5:41747 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41747 2024-11-16T12:51:21,228 INFO [M:0;0450ab8807f5:41747 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T12:51:21,228 INFO [M:0;0450ab8807f5:41747 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T12:51:21,281 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 0450ab8807f5:41747 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0450ab8807f5/172.17.0.2:41747 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:21,282 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] ipc.FailedServers(52): Added failed server with address 0450ab8807f5:41747 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0450ab8807f5/172.17.0.2:41747 2024-11-16T12:51:21,285 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 0450ab8807f5:41747 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0450ab8807f5/172.17.0.2:41747 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:21,286 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] ipc.FailedServers(52): Added failed server with address 0450ab8807f5:41747 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0450ab8807f5/172.17.0.2:41747 2024-11-16T12:51:21,384 DEBUG [RS:0;0450ab8807f5:46655 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T12:51:21,387 INFO [HMaster-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60339, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T12:51:21,388 DEBUG [RS:1;0450ab8807f5:46821 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T12:51:21,388 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:21,389 INFO [HMaster-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58803, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T12:51:21,390 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:21,491 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:21,493 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:21,594 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:21,596 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:21,696 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:21,699 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:21,799 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:21,801 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:21,856 INFO [Time-limited test-EventThread {}] hbase.HBaseTestingUtil$1(2484): Monitor ZKW received event=WatchedEvent state:Expired type:None path:null 2024-11-16T12:51:21,902 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:21,904 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:21,957 INFO [Time-limited test {}] hbase.TestZooKeeper(153): Creating table testMasterSessionExpired_1731761481957 2024-11-16T12:51:21,957 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T12:51:21,958 DEBUG [ReadOnlyZKClient-127.0.0.1:53208@0x0b9b1968 {}] client.AsyncConnectionImpl(321): The fetched master address is 0450ab8807f5,43579,1731761477998 2024-11-16T12:51:21,958 DEBUG [ReadOnlyZKClient-127.0.0.1:53208@0x0b9b1968 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@246e2296 2024-11-16T12:51:21,958 DEBUG [ReadOnlyZKClient-127.0.0.1:53208@0x0b9b1968 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T12:51:21,960 INFO [HMaster-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37524, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T12:51:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.checkInitialized(HMaster.java:3340) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.createTable(HMaster.java:2437) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.createTable(MasterRpcServices.java:786) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:22,005 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:22,007 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.checkInitialized(HMaster.java:3340) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.createTable(HMaster.java:2437) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.createTable(MasterRpcServices.java:786) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:22,107 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:22,109 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:22,211 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:22,213 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.checkInitialized(HMaster.java:3340) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.createTable(HMaster.java:2437) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.createTable(MasterRpcServices.java:786) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:22,316 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:22,318 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:22,419 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:22,422 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:22,524 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:22,533 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.checkInitialized(HMaster.java:3340) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.createTable(HMaster.java:2437) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.createTable(MasterRpcServices.java:786) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:22,636 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:22,637 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:22,740 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:22,743 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:22,845 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:22,847 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:22,887 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T12:51:22,949 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:22,951 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:23,054 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:23,056 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:23,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.checkInitialized(HMaster.java:3340) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.createTable(HMaster.java:2437) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.createTable(MasterRpcServices.java:786) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:23,161 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:23,162 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:23,265 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:23,271 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:23,375 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:23,376 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:23,480 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:23,481 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:23,583 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:23,584 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:23,686 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:23,687 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:23,789 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:23,789 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:23,892 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:23,892 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:23,900 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,41747,1731761477856-dead/0450ab8807f5%2C41747%2C1731761477856.1731761478740 after 4002ms 2024-11-16T12:51:23,901 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(328): Renamed hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,41747,1731761477856-dead/0450ab8807f5%2C41747%2C1731761477856.1731761478740 to hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.wals/0450ab8807f5%2C41747%2C1731761477856.1731761478740 2024-11-16T12:51:23,901 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(330): Delete empty local region wal dir hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,41747,1731761477856-dead 2024-11-16T12:51:23,902 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,43579,1731761477998 2024-11-16T12:51:23,903 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T12:51:23,905 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0450ab8807f5%2C43579%2C1731761477998, suffix=, logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,43579,1731761477998, archiveDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/oldWALs, maxLogs=10 2024-11-16T12:51:23,918 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,43579,1731761477998/0450ab8807f5%2C43579%2C1731761477998.1731761483905, exclude list is [], retry=0 2024-11-16T12:51:23,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43257,DS-06d91055-02a4-40d7-bddb-15aee81f02ac,DISK] 2024-11-16T12:51:23,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36961,DS-7ef9e9a6-250e-467c-989b-a9ffcab2299e,DISK] 2024-11-16T12:51:23,923 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,43579,1731761477998/0450ab8807f5%2C43579%2C1731761477998.1731761483905 2024-11-16T12:51:23,924 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:46667:46667)] 2024-11-16T12:51:23,924 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-16T12:51:23,924 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:23,924 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:23,924 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:23,926 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:23,927 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-16T12:51:23,927 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:23,928 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:23,928 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:23,929 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-16T12:51:23,929 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:23,930 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:23,930 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:23,931 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-16T12:51:23,931 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:23,931 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:23,932 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:23,932 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-16T12:51:23,932 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:23,933 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:23,933 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:23,934 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(5516): Found 1 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.wals 2024-11-16T12:51:23,934 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.wals/0450ab8807f5%2C41747%2C1731761477856.1731761478740 2024-11-16T12:51:23,941 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(5793): Applied 36, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=28, path=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.wals/0450ab8807f5%2C41747%2C1731761477856.1731761478740 2024-11-16T12:51:23,941 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.68 KB heapSize=11.36 KB 2024-11-16T12:51:23,961 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5a7c1c3de4aa489fa0d8dde845973978 is 82, key is hbase:meta,,1/info:regioninfo/1731761479681/Put/seqid=0 2024-11-16T12:51:23,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741856_1034 (size=5672) 2024-11-16T12:51:23,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741856_1034 (size=5672) 2024-11-16T12:51:23,968 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5a7c1c3de4aa489fa0d8dde845973978 2024-11-16T12:51:23,991 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ab4cb2e39b9c47db8e1fa7ecf9559c36 is 366, key is \x00\x00\x00\x00\x00\x00\x00\x02/proc:d/1731761479718/Put/seqid=0 2024-11-16T12:51:23,995 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:23,995 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:23,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741857_1035 (size=12229) 2024-11-16T12:51:23,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741857_1035 (size=12229) 2024-11-16T12:51:23,998 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ab4cb2e39b9c47db8e1fa7ecf9559c36 2024-11-16T12:51:24,022 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0344c552ac8947c780b789052d7c7236 is 69, key is 0450ab8807f5,46655,1731761478065/rs:state/1731761479102/Put/seqid=0 2024-11-16T12:51:24,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741858_1036 (size=5224) 2024-11-16T12:51:24,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741858_1036 (size=5224) 2024-11-16T12:51:24,030 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0344c552ac8947c780b789052d7c7236 2024-11-16T12:51:24,040 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5a7c1c3de4aa489fa0d8dde845973978 as hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5a7c1c3de4aa489fa0d8dde845973978 2024-11-16T12:51:24,049 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5a7c1c3de4aa489fa0d8dde845973978, entries=8, sequenceid=28, filesize=5.5 K 2024-11-16T12:51:24,050 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ab4cb2e39b9c47db8e1fa7ecf9559c36 as hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ab4cb2e39b9c47db8e1fa7ecf9559c36 2024-11-16T12:51:24,059 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ab4cb2e39b9c47db8e1fa7ecf9559c36, entries=26, sequenceid=28, filesize=11.9 K 2024-11-16T12:51:24,060 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0344c552ac8947c780b789052d7c7236 as hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0344c552ac8947c780b789052d7c7236 2024-11-16T12:51:24,067 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0344c552ac8947c780b789052d7c7236, entries=2, sequenceid=28, filesize=5.1 K 2024-11-16T12:51:24,067 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.68 KB/7867, heapSize ~11.06 KB/11328, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 126ms, sequenceid=28, compaction requested=false; wal=null 2024-11-16T12:51:24,068 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.wals/0450ab8807f5%2C41747%2C1731761477856.1731761478740 2024-11-16T12:51:24,070 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:24,070 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:24,071 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T12:51:24,072 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:24,075 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/28.seqid, newMaxSeqId=28, maxSeqId=1 2024-11-16T12:51:24,076 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=29; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66860662, jitterRate=-0.003698498010635376}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T12:51:24,076 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731761483924Initializing all the Stores at 1731761483926 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761483926Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761483926Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761483926Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761483926Obtaining lock to block concurrent updates at 1731761483941 (+15 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731761483941Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7867, getHeapSize=11568, getOffHeapSize=0, getCellsCount=36 at 1731761483941Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731761483941Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731761483941Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731761483960 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731761483961 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731761483976 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731761483990 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731761483990Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731761484007 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731761484022 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731761484022Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c052acb: reopening flushed file at 1731761484039 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b9531d2: reopening flushed file at 1731761484049 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ddc2235: reopening flushed file at 1731761484059 (+10 ms)Finished flush of dataSize ~7.68 KB/7867, heapSize ~11.06 KB/11328, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 126ms, sequenceid=28, compaction requested=false; wal=null at 1731761484067 (+8 ms)Cleaning up temporary data from old regions at 1731761484070 (+3 ms)Region opened successfully at 1731761484076 (+6 ms) 2024-11-16T12:51:24,077 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-16T12:51:24,080 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1077f794, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0450ab8807f5/172.17.0.2:0 2024-11-16T12:51:24,089 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(884): The info family in master local region already has data in it, skip migrating... 2024-11-16T12:51:24,090 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-16T12:51:24,090 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-16T12:51:24,090 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T12:51:24,091 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T12:51:24,095 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T12:51:24,095 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-16T12:51:24,095 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-16T12:51:24,097 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:24,098 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:24,099 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=0450ab8807f5,46821,1731761478114, regionLocation=0450ab8807f5,46821,1731761478114, openSeqNum=2 2024-11-16T12:51:24,099 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=0450ab8807f5,46821,1731761478114, table=hbase:meta, region=1588230740 2024-11-16T12:51:24,100 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0450ab8807f5,46821,1731761478114, state=OPEN 2024-11-16T12:51:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.checkInitialized(HMaster.java:3340) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.createTable(HMaster.java:2437) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.createTable(MasterRpcServices.java:786) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:24,110 WARN [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRpcRetryingCaller(177): Call to master failed, tries = 6, maxAttempts = 15, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.HMaster.checkInitialized(HMaster.java:3340) at org.apache.hadoop.hbase.master.HMaster.createTable(HMaster.java:2437) at org.apache.hadoop.hbase.master.MasterRpcServices.createTable(MasterRpcServices.java:786) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor58.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:465) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.HMaster.checkInitialized(HMaster.java:3340) at org.apache.hadoop.hbase.master.HMaster.createTable(HMaster.java:2437) at org.apache.hadoop.hbase.master.MasterRpcServices.createTable(MasterRpcServices.java:786) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T12:51:24,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46655-0x10144fc6ca60008, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:24,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:24,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46821-0x10144fc6ca60009, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:24,155 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:24,155 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:24,155 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:24,159 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 2 possibly 'live' servers, and 0 'splitting'. 2024-11-16T12:51:24,160 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ServerManager(517): Registering regionserver=0450ab8807f5,46821,1731761478114 2024-11-16T12:51:24,161 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ServerManager(517): Registering regionserver=0450ab8807f5,46655,1731761478065 2024-11-16T12:51:24,162 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-16T12:51:24,171 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-16T12:51:24,172 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-16T12:51:24,172 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-16T12:51:24,179 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-16T12:51:24,180 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-16T12:51:24,181 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-16T12:51:24,188 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-16T12:51:24,189 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-16T12:51:24,196 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-16T12:51:24,198 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-16T12:51:24,200 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:24,200 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:24,204 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-16T12:51:24,205 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0450ab8807f5,43579,1731761477998, sessionid=0x10144fc6ca60007, setting cluster-up flag (Was=true) 2024-11-16T12:51:24,213 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-16T12:51:24,214 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0450ab8807f5,43579,1731761477998 2024-11-16T12:51:24,222 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-16T12:51:24,223 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0450ab8807f5,43579,1731761477998 2024-11-16T12:51:24,224 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ServerManager(1190): begin to load .lastflushedseqids at hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/.lastflushedseqids 2024-11-16T12:51:24,228 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-16T12:51:24,228 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0450ab8807f5,43579,1731761477998 Number of backup masters: 0 Number of live region servers: 2 0450ab8807f5,46655,1731761478065 0450ab8807f5,46821,1731761478114 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-16T12:51:24,230 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:24,230 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:24,230 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:24,230 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:24,230 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0450ab8807f5:0, corePoolSize=10, maxPoolSize=10 2024-11-16T12:51:24,230 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:24,230 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0450ab8807f5:0, corePoolSize=2, maxPoolSize=2 2024-11-16T12:51:24,230 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:24,231 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731761514231 2024-11-16T12:51:24,231 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-16T12:51:24,231 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-16T12:51:24,231 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-16T12:51:24,231 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-16T12:51:24,231 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-16T12:51:24,231 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-16T12:51:24,232 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:24,232 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-16T12:51:24,232 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-16T12:51:24,232 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-16T12:51:24,232 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-16T12:51:24,232 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-16T12:51:24,232 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.large.0-1731761484232,5,FailOnTimeoutGroup] 2024-11-16T12:51:24,233 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.small.0-1731761484232,5,FailOnTimeoutGroup] 2024-11-16T12:51:24,233 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:24,233 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-16T12:51:24,233 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:24,233 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:24,233 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731761484233, completionTime=-1 2024-11-16T12:51:24,233 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-16T12:51:24,233 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T12:51:24,233 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T12:51:24,233 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,46821,1731761478114, seqNum=-1] 2024-11-16T12:51:24,234 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T12:51:24,235 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32787, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T12:51:24,237 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T12:51:24,237 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731761544237 2024-11-16T12:51:24,237 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731761604237 2024-11-16T12:51:24,237 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec 2024-11-16T12:51:24,238 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,43579,1731761477998-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:24,238 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,43579,1731761477998-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:24,238 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,43579,1731761477998-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:24,238 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0450ab8807f5:43579, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:24,238 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:24,238 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:24,240 DEBUG [master/0450ab8807f5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-16T12:51:24,243 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.404sec 2024-11-16T12:51:24,243 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-16T12:51:24,243 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-16T12:51:24,243 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-16T12:51:24,243 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-16T12:51:24,243 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-16T12:51:24,244 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,43579,1731761477998-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T12:51:24,244 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,43579,1731761477998-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-16T12:51:24,247 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-16T12:51:24,247 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-16T12:51:24,247 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,43579,1731761477998-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:24,744 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionServerSessionExpired_1731761468936 2024-11-16T12:51:25,128 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-16T12:51:26,120 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43579 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testMasterSessionExpired_1731761481957', {NAME => 'fam', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T12:51:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43579 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testMasterSessionExpired_1731761481957 2024-11-16T12:51:26,125 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testMasterSessionExpired_1731761481957 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T12:51:26,125 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:26,125 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43579 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testMasterSessionExpired_1731761481957" procId is: 4 2024-11-16T12:51:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43579 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T12:51:26,127 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testMasterSessionExpired_1731761481957 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T12:51:26,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741859_1037 (size=389) 2024-11-16T12:51:26,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741859_1037 (size=389) 2024-11-16T12:51:26,138 INFO [RegionOpenAndInit-testMasterSessionExpired_1731761481957-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9f507782751266ab2af43bdbba26d830, NAME => 'testMasterSessionExpired_1731761481957,,1731761486120.9f507782751266ab2af43bdbba26d830.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testMasterSessionExpired_1731761481957', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'fam', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:26,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741860_1038 (size=73) 2024-11-16T12:51:26,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741860_1038 (size=73) 2024-11-16T12:51:26,147 DEBUG [RegionOpenAndInit-testMasterSessionExpired_1731761481957-pool-0 {}] regionserver.HRegion(898): Instantiated testMasterSessionExpired_1731761481957,,1731761486120.9f507782751266ab2af43bdbba26d830.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:26,147 DEBUG [RegionOpenAndInit-testMasterSessionExpired_1731761481957-pool-0 {}] regionserver.HRegion(1722): Closing 9f507782751266ab2af43bdbba26d830, disabling compactions & flushes 2024-11-16T12:51:26,147 INFO [RegionOpenAndInit-testMasterSessionExpired_1731761481957-pool-0 {}] regionserver.HRegion(1755): Closing region testMasterSessionExpired_1731761481957,,1731761486120.9f507782751266ab2af43bdbba26d830. 2024-11-16T12:51:26,147 DEBUG [RegionOpenAndInit-testMasterSessionExpired_1731761481957-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testMasterSessionExpired_1731761481957,,1731761486120.9f507782751266ab2af43bdbba26d830. 2024-11-16T12:51:26,147 DEBUG [RegionOpenAndInit-testMasterSessionExpired_1731761481957-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testMasterSessionExpired_1731761481957,,1731761486120.9f507782751266ab2af43bdbba26d830. after waiting 0 ms 2024-11-16T12:51:26,147 DEBUG [RegionOpenAndInit-testMasterSessionExpired_1731761481957-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testMasterSessionExpired_1731761481957,,1731761486120.9f507782751266ab2af43bdbba26d830. 2024-11-16T12:51:26,147 INFO [RegionOpenAndInit-testMasterSessionExpired_1731761481957-pool-0 {}] regionserver.HRegion(1973): Closed testMasterSessionExpired_1731761481957,,1731761486120.9f507782751266ab2af43bdbba26d830. 2024-11-16T12:51:26,147 DEBUG [RegionOpenAndInit-testMasterSessionExpired_1731761481957-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9f507782751266ab2af43bdbba26d830: Waiting for close lock at 1731761486147Disabling compacts and flushes for region at 1731761486147Disabling writes for close at 1731761486147Writing region close event to WAL at 1731761486147Closed at 1731761486147 2024-11-16T12:51:26,149 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testMasterSessionExpired_1731761481957 execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T12:51:26,150 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testMasterSessionExpired_1731761481957,,1731761486120.9f507782751266ab2af43bdbba26d830.","families":{"info":[{"qualifier":"regioninfo","vlen":72,"tag":[],"timestamp":"1731761486149"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731761486149"}]},"ts":"1731761486149"} 2024-11-16T12:51:26,152 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T12:51:26,154 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testMasterSessionExpired_1731761481957 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T12:51:26,154 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testMasterSessionExpired_1731761481957","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731761486154"}]},"ts":"1731761486154"} 2024-11-16T12:51:26,157 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testMasterSessionExpired_1731761481957, state=ENABLING in hbase:meta 2024-11-16T12:51:26,157 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {0450ab8807f5=0} racks are {/default-rack=0} 2024-11-16T12:51:26,158 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T12:51:26,158 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T12:51:26,158 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T12:51:26,158 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T12:51:26,158 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T12:51:26,158 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T12:51:26,158 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T12:51:26,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testMasterSessionExpired_1731761481957, region=9f507782751266ab2af43bdbba26d830, ASSIGN}] 2024-11-16T12:51:26,160 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testMasterSessionExpired_1731761481957, region=9f507782751266ab2af43bdbba26d830, ASSIGN 2024-11-16T12:51:26,161 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testMasterSessionExpired_1731761481957, region=9f507782751266ab2af43bdbba26d830, ASSIGN; state=OFFLINE, location=0450ab8807f5,46655,1731761478065; forceNewPlan=false, retain=false 2024-11-16T12:51:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43579 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T12:51:26,314 INFO [0450ab8807f5:43579 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-16T12:51:26,315 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9f507782751266ab2af43bdbba26d830, regionState=OPENING, regionLocation=0450ab8807f5,46655,1731761478065 2024-11-16T12:51:26,319 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testMasterSessionExpired_1731761481957, region=9f507782751266ab2af43bdbba26d830, ASSIGN because future has completed 2024-11-16T12:51:26,320 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9f507782751266ab2af43bdbba26d830, server=0450ab8807f5,46655,1731761478065}] 2024-11-16T12:51:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43579 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T12:51:26,474 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T12:51:26,476 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58199, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T12:51:26,481 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testMasterSessionExpired_1731761481957,,1731761486120.9f507782751266ab2af43bdbba26d830. 2024-11-16T12:51:26,481 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 9f507782751266ab2af43bdbba26d830, NAME => 'testMasterSessionExpired_1731761481957,,1731761486120.9f507782751266ab2af43bdbba26d830.', STARTKEY => '', ENDKEY => ''} 2024-11-16T12:51:26,482 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMasterSessionExpired_1731761481957 9f507782751266ab2af43bdbba26d830 2024-11-16T12:51:26,482 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testMasterSessionExpired_1731761481957,,1731761486120.9f507782751266ab2af43bdbba26d830.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:26,482 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 9f507782751266ab2af43bdbba26d830 2024-11-16T12:51:26,482 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 9f507782751266ab2af43bdbba26d830 2024-11-16T12:51:26,484 INFO [StoreOpener-9f507782751266ab2af43bdbba26d830-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family fam of region 9f507782751266ab2af43bdbba26d830 2024-11-16T12:51:26,486 INFO [StoreOpener-9f507782751266ab2af43bdbba26d830-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9f507782751266ab2af43bdbba26d830 columnFamilyName fam 2024-11-16T12:51:26,486 DEBUG [StoreOpener-9f507782751266ab2af43bdbba26d830-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:26,486 INFO [StoreOpener-9f507782751266ab2af43bdbba26d830-1 {}] regionserver.HStore(327): Store=9f507782751266ab2af43bdbba26d830/fam, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:26,487 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 9f507782751266ab2af43bdbba26d830 2024-11-16T12:51:26,488 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testMasterSessionExpired_1731761481957/9f507782751266ab2af43bdbba26d830 2024-11-16T12:51:26,488 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testMasterSessionExpired_1731761481957/9f507782751266ab2af43bdbba26d830 2024-11-16T12:51:26,489 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 9f507782751266ab2af43bdbba26d830 2024-11-16T12:51:26,489 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 9f507782751266ab2af43bdbba26d830 2024-11-16T12:51:26,491 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 9f507782751266ab2af43bdbba26d830 2024-11-16T12:51:26,494 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testMasterSessionExpired_1731761481957/9f507782751266ab2af43bdbba26d830/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:51:26,495 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 9f507782751266ab2af43bdbba26d830; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66479561, jitterRate=-0.009377345442771912}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T12:51:26,495 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9f507782751266ab2af43bdbba26d830 2024-11-16T12:51:26,496 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 9f507782751266ab2af43bdbba26d830: Running coprocessor pre-open hook at 1731761486482Writing region info on filesystem at 1731761486482Initializing all the Stores at 1731761486483 (+1 ms)Instantiating store for column family {NAME => 'fam', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761486484 (+1 ms)Cleaning up temporary data from old regions at 1731761486489 (+5 ms)Running coprocessor post-open hooks at 1731761486495 (+6 ms)Region opened successfully at 1731761486496 (+1 ms) 2024-11-16T12:51:26,498 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testMasterSessionExpired_1731761481957,,1731761486120.9f507782751266ab2af43bdbba26d830., pid=6, masterSystemTime=1731761486473 2024-11-16T12:51:26,501 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testMasterSessionExpired_1731761481957,,1731761486120.9f507782751266ab2af43bdbba26d830. 2024-11-16T12:51:26,501 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testMasterSessionExpired_1731761481957,,1731761486120.9f507782751266ab2af43bdbba26d830. 2024-11-16T12:51:26,502 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9f507782751266ab2af43bdbba26d830, regionState=OPEN, openSeqNum=2, regionLocation=0450ab8807f5,46655,1731761478065 2024-11-16T12:51:26,505 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9f507782751266ab2af43bdbba26d830, server=0450ab8807f5,46655,1731761478065 because future has completed 2024-11-16T12:51:26,510 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-16T12:51:26,511 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 9f507782751266ab2af43bdbba26d830, server=0450ab8807f5,46655,1731761478065 in 187 msec 2024-11-16T12:51:26,514 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-16T12:51:26,514 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testMasterSessionExpired_1731761481957, region=9f507782751266ab2af43bdbba26d830, ASSIGN in 352 msec 2024-11-16T12:51:26,515 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testMasterSessionExpired_1731761481957 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T12:51:26,516 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testMasterSessionExpired_1731761481957","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731761486515"}]},"ts":"1731761486515"} 2024-11-16T12:51:26,519 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testMasterSessionExpired_1731761481957, state=ENABLED in hbase:meta 2024-11-16T12:51:26,520 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testMasterSessionExpired_1731761481957 execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T12:51:26,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testMasterSessionExpired_1731761481957 in 399 msec 2024-11-16T12:51:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43579 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T12:51:26,760 INFO [RPCClient-NioEventLoopGroup-6-5 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testMasterSessionExpired_1731761481957 completed 2024-11-16T12:51:26,760 INFO [Time-limited test {}] hbase.TestZooKeeper(164): Putting table testMasterSessionExpired_1731761481957 2024-11-16T12:51:26,764 DEBUG [RPCClient-NioEventLoopGroup-6-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testMasterSessionExpired_1731761481957', row='testrow', locateType=CURRENT is [region=testMasterSessionExpired_1731761481957,,1731761486120.9f507782751266ab2af43bdbba26d830., hostname=0450ab8807f5,46655,1731761478065, seqNum=2] 2024-11-16T12:51:26,764 DEBUG [RPCClient-NioEventLoopGroup-6-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T12:51:26,766 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41318, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T12:51:26,768 DEBUG [Time-limited test {}] master.HMaster(3213): Abort called but aborted=true, stopped=true 2024-11-16T12:51:26,768 ERROR [Time-limited test {}] master.HMaster(3218): Master server abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-16T12:51:26,769 ERROR [Time-limited test {}] master.HMaster(3225): ***** ABORTING master 0450ab8807f5,43579,1731761477998: killAll ***** 2024-11-16T12:51:26,769 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master '0450ab8807f5,43579,1731761477998' ***** 2024-11-16T12:51:26,769 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: Stopped by Time-limited test 2024-11-16T12:51:26,769 INFO [M:1;0450ab8807f5:43579 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T12:51:26,769 INFO [M:1;0450ab8807f5:43579 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:1;0450ab8807f5:43579. 2024-11-16T12:51:26,769 DEBUG [M:1;0450ab8807f5:43579 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:51:26,769 DEBUG [M:1;0450ab8807f5:43579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:26,769 INFO [M:1;0450ab8807f5:43579 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T12:51:26,769 DEBUG [M:1;0450ab8807f5:43579 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T12:51:26,769 DEBUG [M:1;0450ab8807f5:43579 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T12:51:26,769 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T12:51:26,769 DEBUG [master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.large.0-1731761484232 {}] cleaner.HFileCleaner(306): Exit Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.large.0-1731761484232,5,FailOnTimeoutGroup] 2024-11-16T12:51:26,769 DEBUG [master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.small.0-1731761484232 {}] cleaner.HFileCleaner(306): Exit Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.small.0-1731761484232,5,FailOnTimeoutGroup] 2024-11-16T12:51:26,769 INFO [M:1;0450ab8807f5:43579 {}] hbase.ChoreService(370): Chore service for: master/0450ab8807f5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T12:51:26,770 INFO [M:1;0450ab8807f5:43579 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T12:51:26,770 DEBUG [M:1;0450ab8807f5:43579 {}] master.HMaster(1795): Stopping service threads 2024-11-16T12:51:26,770 INFO [M:1;0450ab8807f5:43579 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T12:51:26,770 INFO [M:1;0450ab8807f5:43579 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T12:51:26,770 ERROR [M:1;0450ab8807f5:43579 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] 2024-11-16T12:51:26,770 INFO [M:1;0450ab8807f5:43579 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=true 2024-11-16T12:51:26,770 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T12:51:26,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:26,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46821-0x10144fc6ca60009, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:26,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:26,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46655-0x10144fc6ca60008, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:26,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46821-0x10144fc6ca60009, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:26,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46655-0x10144fc6ca60008, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:26,821 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 0450ab8807f5,46655,1731761478065: killAll ***** 2024-11-16T12:51:26,821 DEBUG [M:1;0450ab8807f5:43579 {}] zookeeper.ZKUtil(347): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-16T12:51:26,821 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-16T12:51:26,822 WARN [M:1;0450ab8807f5:43579 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-16T12:51:26,822 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-16T12:51:26,822 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46655-0x10144fc6ca60008, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:26,822 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46821-0x10144fc6ca60009, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:26,823 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-16T12:51:26,823 INFO [M:1;0450ab8807f5:43579 {}] master.ServerManager(1134): Rewriting .lastflushedseqids file at: hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/.lastflushedseqids 2024-11-16T12:51:26,826 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-16T12:51:26,826 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-16T12:51:26,833 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 464966912 }, "NonHeapMemoryUsage": { "committed": 158597120, "init": 7667712, "max": -1, "used": 156107072 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "0450ab8807f5", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 7887, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 27, "ProcessCallTime_min": 0, "ProcessCallTime_max": 6, "ProcessCallTime_mean": 1, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 1, "ProcessCallTime_75th_percentile": 2, "ProcessCallTime_90th_percentile": 6, "ProcessCallTime_95th_percentile": 6, "ProcessCallTime_98th_percentile": 6, "ProcessCallTime_99th_percentile": 6, "ProcessCallTime_99.9th_percentile": 6, "ProcessCallTime_TimeRangeCount_0-1": 5, "ProcessCallTime_TimeRangeCount_1-3": 2, "ProcessCallTime_TimeRangeCount_3-10": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 27, "QueueCallTime_min": 0, "QueueCallTime_max": 0, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 27, "TotalCallTime_min": 0, "TotalCallTime_max": 6, "TotalCallTime_mean": 1, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 1, "TotalCallTime_75th_percentile": 2, "TotalCallTime_90th_percentile": 6, "TotalCallTime_95th_percentile": 6, "TotalCallTime_98th_percentile": 6, "TotalCallTime_99th_percentile": 6, "TotalCallTime_99.9th_percentile": 6, "TotalCallTime_TimeRangeCount_0-1": 5, "TotalCallTime_TimeRangeCount_1-3": 2, "TotalCallTime_TimeRangeCount_3-10": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 27, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 4, "ResponseSize_75th_percentile": 25, "ResponseSize_90th_percentile": 173, "ResponseSize_95th_percentile": 173, "ResponseSize_98th_percentile": 173, "ResponseSize_99th_percentile": 173, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 4, "ResponseSize_SizeRangeCount_10-100": 2, "ResponseSize_SizeRangeCount_100-1000": 2, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 27, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 1755 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "0450ab8807f5", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:53208", "tag.serverName": "0450ab8807f5,36743,1731761465323", "tag.clusterId": "be09e536-22d9-488d-a733-7968a5bb48e1", "tag.Context": "regionserver", "tag.Hostname": "0450ab8807f5", "regionCount": 0, "storeCount": 0, "hlogFileCount": 5, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": -1030.6, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731761465323, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 1836, "localBytesRead": 1836, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 2100, "ByteBuffAllocatorPoolAllocationBytes": 66560, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 7, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 1, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 6, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 1, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 6, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 5153, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 74, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 1, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-16T12:51:26,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741861_1039 (size=147) 2024-11-16T12:51:26,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741861_1039 (size=147) 2024-11-16T12:51:26,835 DEBUG [HMaster-EventLoopGroup-8-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0450ab8807f5,43579,1731761477998 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:26,835 WARN [Time-limited test {}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.regionserver.RegionServerAbortedException): Server 0450ab8807f5,43579,1731761477998 aborting at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:?] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:?] at org.apache.hadoop.hbase.Abortable.abort(Abortable.java:45) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster.killAll(SingleProcessHBaseCluster.java:880) ~[test-classes/:?] at org.apache.hadoop.hbase.TestZooKeeper.after(TestZooKeeper.java:107) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: Server 0450ab8807f5,43579,1731761477998 aborting at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-16T12:51:26,836 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0450ab8807f5,46655,1731761478065' ***** 2024-11-16T12:51:26,836 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: killAll 2024-11-16T12:51:26,836 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 0450ab8807f5,46821,1731761478114: killAll ***** 2024-11-16T12:51:26,836 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-16T12:51:26,836 INFO [RS:0;0450ab8807f5:46655 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T12:51:26,836 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-16T12:51:26,837 INFO [RS:0;0450ab8807f5:46655 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-16T12:51:26,837 INFO [RS:0;0450ab8807f5:46655 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-16T12:51:26,837 INFO [RS:0;0450ab8807f5:46655 {}] regionserver.HRegionServer(3091): Received CLOSE for 9f507782751266ab2af43bdbba26d830 2024-11-16T12:51:26,837 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T12:51:26,837 INFO [RS:0;0450ab8807f5:46655 {}] regionserver.HRegionServer(956): aborting server 0450ab8807f5,46655,1731761478065 2024-11-16T12:51:26,837 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-16T12:51:26,837 INFO [RS:0;0450ab8807f5:46655 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T12:51:26,837 INFO [RS:0;0450ab8807f5:46655 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0450ab8807f5:46655. 2024-11-16T12:51:26,837 DEBUG [RS:0;0450ab8807f5:46655 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:51:26,837 DEBUG [RS:0;0450ab8807f5:46655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:26,838 INFO [RS:0;0450ab8807f5:46655 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-16T12:51:26,838 DEBUG [RS:0;0450ab8807f5:46655 {}] regionserver.HRegionServer(1325): Online Regions={9f507782751266ab2af43bdbba26d830=testMasterSessionExpired_1731761481957,,1731761486120.9f507782751266ab2af43bdbba26d830.} 2024-11-16T12:51:26,838 DEBUG [RS:0;0450ab8807f5:46655 {}] regionserver.HRegionServer(1351): Waiting on 9f507782751266ab2af43bdbba26d830 2024-11-16T12:51:26,838 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-16T12:51:26,838 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9f507782751266ab2af43bdbba26d830, disabling compactions & flushes 2024-11-16T12:51:26,838 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testMasterSessionExpired_1731761481957,,1731761486120.9f507782751266ab2af43bdbba26d830. 2024-11-16T12:51:26,838 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testMasterSessionExpired_1731761481957,,1731761486120.9f507782751266ab2af43bdbba26d830. 2024-11-16T12:51:26,838 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testMasterSessionExpired_1731761481957,,1731761486120.9f507782751266ab2af43bdbba26d830. after waiting 0 ms 2024-11-16T12:51:26,838 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testMasterSessionExpired_1731761481957,,1731761486120.9f507782751266ab2af43bdbba26d830. 2024-11-16T12:51:26,839 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-16T12:51:26,841 INFO [M:1;0450ab8807f5:43579 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T12:51:26,841 INFO [M:1;0450ab8807f5:43579 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=true 2024-11-16T12:51:26,842 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 467064064 }, "NonHeapMemoryUsage": { "committed": 158793728, "init": 7667712, "max": -1, "used": 156237400 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "0450ab8807f5", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 7887, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 27, "ProcessCallTime_min": 0, "ProcessCallTime_max": 6, "ProcessCallTime_mean": 1, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 1, "ProcessCallTime_75th_percentile": 2, "ProcessCallTime_90th_percentile": 6, "ProcessCallTime_95th_percentile": 6, "ProcessCallTime_98th_percentile": 6, "ProcessCallTime_99th_percentile": 6, "ProcessCallTime_99.9th_percentile": 6, "ProcessCallTime_TimeRangeCount_0-1": 5, "ProcessCallTime_TimeRangeCount_1-3": 2, "ProcessCallTime_TimeRangeCount_3-10": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 27, "QueueCallTime_min": 0, "QueueCallTime_max": 0, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 27, "TotalCallTime_min": 0, "TotalCallTime_max": 6, "TotalCallTime_mean": 1, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 1, "TotalCallTime_75th_percentile": 2, "TotalCallTime_90th_percentile": 6, "TotalCallTime_95th_percentile": 6, "TotalCallTime_98th_percentile": 6, "TotalCallTime_99th_percentile": 6, "TotalCallTime_99.9th_percentile": 6, "TotalCallTime_TimeRangeCount_0-1": 5, "TotalCallTime_TimeRangeCount_1-3": 2, "TotalCallTime_TimeRangeCount_3-10": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 27, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 4, "ResponseSize_75th_percentile": 25, "ResponseSize_90th_percentile": 173, "ResponseSize_95th_percentile": 173, "ResponseSize_98th_percentile": 173, "ResponseSize_99th_percentile": 173, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 4, "ResponseSize_SizeRangeCount_10-100": 2, "ResponseSize_SizeRangeCount_100-1000": 2, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 27, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 1755 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "0450ab8807f5", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:53208", "tag.serverName": "0450ab8807f5,36743,1731761465323", "tag.clusterId": "be09e536-22d9-488d-a733-7968a5bb48e1", "tag.Context": "regionserver", "tag.Hostname": "0450ab8807f5", "regionCount": 0, "storeCount": 0, "hlogFileCount": 5, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": -1030.6, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731761465323, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 1836, "localBytesRead": 1836, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 2100, "ByteBuffAllocatorPoolAllocationBytes": 66560, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 7, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 1, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 6, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 1, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 6, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 5153, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 74, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 1, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-16T12:51:26,843 INFO [regionserver/0450ab8807f5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:51:26,844 ERROR [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1960): Memstore data size is 41 in region testMasterSessionExpired_1731761481957,,1731761486120.9f507782751266ab2af43bdbba26d830. 2024-11-16T12:51:26,844 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testMasterSessionExpired_1731761481957,,1731761486120.9f507782751266ab2af43bdbba26d830. 2024-11-16T12:51:26,844 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9f507782751266ab2af43bdbba26d830: Waiting for close lock at 1731761486838Running coprocessor pre-close hooks at 1731761486838Disabling compacts and flushes for region at 1731761486838Disabling writes for close at 1731761486838Writing region close event to WAL at 1731761486844 (+6 ms)Running coprocessor post-close hooks at 1731761486844Closed at 1731761486844 2024-11-16T12:51:26,844 DEBUG [HMaster-EventLoopGroup-8-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0450ab8807f5,43579,1731761477998 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:26,844 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testMasterSessionExpired_1731761481957,,1731761486120.9f507782751266ab2af43bdbba26d830. 2024-11-16T12:51:26,845 WARN [Time-limited test {}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.regionserver.RegionServerAbortedException): Server 0450ab8807f5,43579,1731761477998 aborting at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:?] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:?] at org.apache.hadoop.hbase.Abortable.abort(Abortable.java:45) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster.killAll(SingleProcessHBaseCluster.java:880) ~[test-classes/:?] at org.apache.hadoop.hbase.TestZooKeeper.after(TestZooKeeper.java:107) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: Server 0450ab8807f5,43579,1731761477998 aborting at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-16T12:51:26,849 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0450ab8807f5,46821,1731761478114' ***** 2024-11-16T12:51:26,849 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: killAll 2024-11-16T12:51:26,849 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T12:51:26,850 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.TestZooKeeper.after(TestZooKeeper.java:110) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:51:26,850 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:26,850 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0b9b1968 to 127.0.0.1:53208 2024-11-16T12:51:26,850 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-16T12:51:26,850 INFO [RS:1;0450ab8807f5:46821 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T12:51:26,851 INFO [RS:1;0450ab8807f5:46821 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-16T12:51:26,851 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T12:51:26,851 INFO [RS:1;0450ab8807f5:46821 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-16T12:51:26,851 INFO [RS:1;0450ab8807f5:46821 {}] regionserver.HRegionServer(956): aborting server 0450ab8807f5,46821,1731761478114 2024-11-16T12:51:26,851 INFO [RS:1;0450ab8807f5:46821 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T12:51:26,851 INFO [RS:1;0450ab8807f5:46821 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;0450ab8807f5:46821. 2024-11-16T12:51:26,851 DEBUG [RS:1;0450ab8807f5:46821 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:51:26,851 DEBUG [RS:1;0450ab8807f5:46821 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:26,851 INFO [RS:1;0450ab8807f5:46821 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T12:51:26,851 INFO [RS:1;0450ab8807f5:46821 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T12:51:26,851 INFO [RS:1;0450ab8807f5:46821 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T12:51:26,851 INFO [RS:1;0450ab8807f5:46821 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T12:51:26,855 INFO [RS:1;0450ab8807f5:46821 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-16T12:51:26,855 DEBUG [RS:1;0450ab8807f5:46821 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-16T12:51:26,855 DEBUG [RS:1;0450ab8807f5:46821 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T12:51:26,855 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T12:51:26,856 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T12:51:26,856 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T12:51:26,856 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T12:51:26,856 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T12:51:26,856 ERROR [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 1707 in region hbase:meta,,1.1588230740 2024-11-16T12:51:26,856 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T12:51:26,856 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T12:51:26,857 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731761486855Running coprocessor pre-close hooks at 1731761486855Disabling compacts and flushes for region at 1731761486855Disabling writes for close at 1731761486856 (+1 ms)Writing region close event to WAL at 1731761486856Running coprocessor post-close hooks at 1731761486856Closed at 1731761486856 2024-11-16T12:51:26,857 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T12:51:26,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741855_1033 (size=17496) 2024-11-16T12:51:26,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741855_1033 (size=17496) 2024-11-16T12:51:26,859 DEBUG [M:1;0450ab8807f5:43579 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T12:51:26,859 INFO [M:1;0450ab8807f5:43579 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:26,859 DEBUG [M:1;0450ab8807f5:43579 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:26,859 DEBUG [M:1;0450ab8807f5:43579 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T12:51:26,859 DEBUG [M:1;0450ab8807f5:43579 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:26,861 ERROR [M:1;0450ab8807f5:43579 {}] regionserver.HRegion(1960): Memstore data size is 15659 in region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:26,861 INFO [M:1;0450ab8807f5:43579 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:26,861 DEBUG [M:1;0450ab8807f5:43579 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731761486859Disabling compacts and flushes for region at 1731761486859Disabling writes for close at 1731761486859Writing region close event to WAL at 1731761486861 (+2 ms)Closed at 1731761486861 2024-11-16T12:51:26,861 INFO [M:1;0450ab8807f5:43579 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-16T12:51:26,861 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T12:51:26,862 INFO [M:1;0450ab8807f5:43579 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43579 2024-11-16T12:51:26,862 INFO [M:1;0450ab8807f5:43579 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T12:51:26,943 INFO [regionserver/0450ab8807f5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:51:26,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:26,971 INFO [M:1;0450ab8807f5:43579 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T12:51:26,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10144fc6ca60007, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:27,038 INFO [RS:0;0450ab8807f5:46655 {}] regionserver.HRegionServer(976): stopping server 0450ab8807f5,46655,1731761478065; all regions closed. 2024-11-16T12:51:27,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741851_1028 (size=583) 2024-11-16T12:51:27,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741851_1028 (size=583) 2024-11-16T12:51:27,043 DEBUG [RS:0;0450ab8807f5:46655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:27,043 INFO [RS:0;0450ab8807f5:46655 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:51:27,043 INFO [RS:0;0450ab8807f5:46655 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T12:51:27,043 INFO [RS:0;0450ab8807f5:46655 {}] hbase.ChoreService(370): Chore service for: regionserver/0450ab8807f5:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T12:51:27,044 INFO [RS:0;0450ab8807f5:46655 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T12:51:27,044 INFO [RS:0;0450ab8807f5:46655 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T12:51:27,044 INFO [regionserver/0450ab8807f5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T12:51:27,044 INFO [RS:0;0450ab8807f5:46655 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T12:51:27,044 INFO [RS:0;0450ab8807f5:46655 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T12:51:27,044 INFO [RS:0;0450ab8807f5:46655 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46655 2024-11-16T12:51:27,056 INFO [RS:1;0450ab8807f5:46821 {}] regionserver.HRegionServer(976): stopping server 0450ab8807f5,46821,1731761478114; all regions closed. 2024-11-16T12:51:27,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741852_1029 (size=2349) 2024-11-16T12:51:27,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741852_1029 (size=2349) 2024-11-16T12:51:27,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741850_1027 (size=93) 2024-11-16T12:51:27,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741850_1027 (size=93) 2024-11-16T12:51:27,066 DEBUG [RS:1;0450ab8807f5:46821 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:27,066 INFO [RS:1;0450ab8807f5:46821 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:51:27,066 INFO [RS:1;0450ab8807f5:46821 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T12:51:27,066 INFO [RS:1;0450ab8807f5:46821 {}] hbase.ChoreService(370): Chore service for: regionserver/0450ab8807f5:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T12:51:27,066 INFO [RS:1;0450ab8807f5:46821 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T12:51:27,066 INFO [regionserver/0450ab8807f5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T12:51:27,067 INFO [RS:1;0450ab8807f5:46821 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46821 2024-11-16T12:51:27,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46655-0x10144fc6ca60008, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0450ab8807f5,46655,1731761478065 2024-11-16T12:51:27,071 INFO [RS:0;0450ab8807f5:46655 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T12:51:27,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46821-0x10144fc6ca60009, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0450ab8807f5,46821,1731761478114 2024-11-16T12:51:27,079 INFO [RS:1;0450ab8807f5:46821 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T12:51:27,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46655-0x10144fc6ca60008, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:27,188 INFO [RS:0;0450ab8807f5:46655 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T12:51:27,188 INFO [RS:0;0450ab8807f5:46655 {}] regionserver.HRegionServer(1031): Exiting; stopping=0450ab8807f5,46655,1731761478065; zookeeper connection closed. 2024-11-16T12:51:27,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46655-0x10144fc6ca60008, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:27,189 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@22fb5ac {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@22fb5ac 2024-11-16T12:51:27,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46821-0x10144fc6ca60009, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:27,196 INFO [RS:1;0450ab8807f5:46821 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T12:51:27,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46821-0x10144fc6ca60009, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:27,196 INFO [RS:1;0450ab8807f5:46821 {}] regionserver.HRegionServer(1031): Exiting; stopping=0450ab8807f5,46821,1731761478114; zookeeper connection closed. 2024-11-16T12:51:27,197 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@33becae {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@33becae 2024-11-16T12:51:27,197 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 2 master(s) and 2 regionserver(s) complete 2024-11-16T12:51:27,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60005, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:27,313 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:27,317 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=testing utility connecting to ZooKeeper ensemble=127.0.0.1:53208 2024-11-16T12:51:27,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility0x0, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:51:27,347 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): testing utility-0x10144fc6ca6000b connected 2024-11-16T12:51:27,347 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-16T12:51:27,347 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/meta-region-server 2024-11-16T12:51:27,348 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/rs 2024-11-16T12:51:27,348 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/splitWAL 2024-11-16T12:51:27,348 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters 2024-11-16T12:51:27,349 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc 2024-11-16T12:51:27,349 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/draining 2024-11-16T12:51:27,349 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/master-maintenance 2024-11-16T12:51:27,350 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot 2024-11-16T12:51:27,350 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/hbaseid 2024-11-16T12:51:27,350 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/table 2024-11-16T12:51:27,352 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc/abort 2024-11-16T12:51:27,353 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc/acquired 2024-11-16T12:51:27,353 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc/reached 2024-11-16T12:51:27,354 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot/abort 2024-11-16T12:51:27,354 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot/acquired 2024-11-16T12:51:27,355 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot/reached 2024-11-16T12:51:27,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/reached 2024-11-16T12:51:27,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/online-snapshot 2024-11-16T12:51:27,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/acquired 2024-11-16T12:51:27,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/abort 2024-11-16T12:51:27,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/reached 2024-11-16T12:51:27,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/flush-table-proc 2024-11-16T12:51:27,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/acquired 2024-11-16T12:51:27,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/abort 2024-11-16T12:51:27,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/table 2024-11-16T12:51:27,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:27,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/hbaseid 2024-11-16T12:51:27,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot 2024-11-16T12:51:27,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master-maintenance 2024-11-16T12:51:27,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/draining 2024-11-16T12:51:27,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc 2024-11-16T12:51:27,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:27,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/splitWAL 2024-11-16T12:51:27,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs 2024-11-16T12:51:27,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:27,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T12:51:27,367 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase 2024-11-16T12:51:27,374 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestZooKeeper#testMasterSessionExpired Thread=332 (was 297) Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:44975 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/cluster_35bff147-7ae7-a41a-0bf2-848ec5006277/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/cluster_35bff147-7ae7-a41a-0bf2-848ec5006277/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-10-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-10-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1205652929) connection to localhost/127.0.0.1:44975 from jenkins.hfs.3 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: HMaster-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/cluster_35bff147-7ae7-a41a-0bf2-848ec5006277/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1205652929) connection to localhost/127.0.0.1:44975 from jenkins.hfs.2 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1694157692_22 at /127.0.0.1:35944 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-10-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:44975 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_248147600_22 at /127.0.0.1:50302 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=594 (was 546) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=232 (was 256), ProcessCount=11 (was 11), AvailableMemoryMB=3261 (was 3367) 2024-11-16T12:51:27,383 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestZooKeeper#testMasterZKSessionRecoveryFailure Thread=332, OpenFileDescriptor=594, MaxFileDescriptor=1048576, SystemLoadAverage=232, ProcessCount=11, AvailableMemoryMB=3261 2024-11-16T12:51:27,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741862_1040 (size=7) 2024-11-16T12:51:27,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741862_1040 (size=7) 2024-11-16T12:51:27,793 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b with version=8 2024-11-16T12:51:27,793 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/hbase-staging 2024-11-16T12:51:27,795 INFO [Time-limited test {}] client.ConnectionUtils(128): master/0450ab8807f5:0 server-side Connection retries=45 2024-11-16T12:51:27,795 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:27,795 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:27,795 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T12:51:27,795 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:27,795 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T12:51:27,795 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T12:51:27,795 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T12:51:27,796 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34423 2024-11-16T12:51:27,797 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34423 connecting to ZooKeeper ensemble=127.0.0.1:53208 2024-11-16T12:51:27,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:344230x0, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:51:27,830 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34423-0x10144fc6ca6000c connected 2024-11-16T12:51:27,887 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:27,889 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:27,891 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:51:27,892 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b, hbase.cluster.distributed=false 2024-11-16T12:51:27,893 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T12:51:27,894 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34423 2024-11-16T12:51:27,894 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34423 2024-11-16T12:51:27,894 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34423 2024-11-16T12:51:27,894 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34423 2024-11-16T12:51:27,895 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34423 2024-11-16T12:51:27,896 INFO [Time-limited test {}] client.ConnectionUtils(128): master/0450ab8807f5:0 server-side Connection retries=45 2024-11-16T12:51:27,896 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:27,896 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:27,896 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T12:51:27,896 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:27,896 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T12:51:27,896 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T12:51:27,896 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T12:51:27,897 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45065 2024-11-16T12:51:27,898 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45065 connecting to ZooKeeper ensemble=127.0.0.1:53208 2024-11-16T12:51:27,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:450650x0, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:51:27,905 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45065-0x10144fc6ca6000d connected 2024-11-16T12:51:27,916 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:27,918 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:27,920 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45065-0x10144fc6ca6000d, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:51:27,920 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b, hbase.cluster.distributed=false 2024-11-16T12:51:27,922 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45065-0x10144fc6ca6000d, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T12:51:27,924 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45065 2024-11-16T12:51:27,924 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45065 2024-11-16T12:51:27,924 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45065 2024-11-16T12:51:27,924 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45065 2024-11-16T12:51:27,925 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45065 2024-11-16T12:51:27,946 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0450ab8807f5:0 server-side Connection retries=45 2024-11-16T12:51:27,946 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:27,946 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:27,946 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T12:51:27,946 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:27,946 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T12:51:27,946 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T12:51:27,946 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T12:51:27,947 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43297 2024-11-16T12:51:27,949 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43297 connecting to ZooKeeper ensemble=127.0.0.1:53208 2024-11-16T12:51:27,950 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:27,952 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:27,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:432970x0, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:51:27,963 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:432970x0, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:51:27,963 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43297-0x10144fc6ca6000e connected 2024-11-16T12:51:27,963 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T12:51:27,964 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T12:51:27,964 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43297-0x10144fc6ca6000e, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:27,965 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43297-0x10144fc6ca6000e, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T12:51:27,965 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43297 2024-11-16T12:51:27,966 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43297 2024-11-16T12:51:27,966 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43297 2024-11-16T12:51:27,966 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43297 2024-11-16T12:51:27,966 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43297 2024-11-16T12:51:27,981 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0450ab8807f5:0 server-side Connection retries=45 2024-11-16T12:51:27,981 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:27,981 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:27,981 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T12:51:27,981 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:27,981 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T12:51:27,981 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T12:51:27,982 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T12:51:27,982 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45447 2024-11-16T12:51:27,984 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45447 connecting to ZooKeeper ensemble=127.0.0.1:53208 2024-11-16T12:51:27,984 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:27,986 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:27,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:454470x0, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:51:27,996 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45447-0x10144fc6ca6000f, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:51:27,996 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45447-0x10144fc6ca6000f connected 2024-11-16T12:51:27,997 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T12:51:27,997 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T12:51:27,998 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45447-0x10144fc6ca6000f, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:27,999 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45447-0x10144fc6ca6000f, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T12:51:27,999 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45447 2024-11-16T12:51:27,999 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45447 2024-11-16T12:51:28,000 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45447 2024-11-16T12:51:28,000 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45447 2024-11-16T12:51:28,001 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45447 2024-11-16T12:51:28,011 DEBUG [M:0;0450ab8807f5:34423 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0450ab8807f5:34423 2024-11-16T12:51:28,011 DEBUG [M:1;0450ab8807f5:45065 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:1;0450ab8807f5:45065 2024-11-16T12:51:28,012 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0450ab8807f5,34423,1731761487794 2024-11-16T12:51:28,012 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0450ab8807f5,45065,1731761487895 2024-11-16T12:51:28,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:28,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43297-0x10144fc6ca6000e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:28,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10144fc6ca6000d, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:28,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45447-0x10144fc6ca6000f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:28,021 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45065-0x10144fc6ca6000d, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0450ab8807f5,45065,1731761487895 2024-11-16T12:51:28,021 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0450ab8807f5,34423,1731761487794 2024-11-16T12:51:28,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43297-0x10144fc6ca6000e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:28,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10144fc6ca6000d, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:28,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45447-0x10144fc6ca6000f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:28,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:28,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43297-0x10144fc6ca6000e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:28,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45447-0x10144fc6ca6000f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:28,038 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45065-0x10144fc6ca6000d, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T12:51:28,038 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T12:51:28,038 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0450ab8807f5,45065,1731761487895 from backup master directory 2024-11-16T12:51:28,038 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ActiveMasterManager(296): Another master is the active master, 0450ab8807f5,45065,1731761487895; waiting to become the next active master 2024-11-16T12:51:28,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10144fc6ca6000d, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0450ab8807f5,45065,1731761487895 2024-11-16T12:51:28,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43297-0x10144fc6ca6000e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:28,046 WARN [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T12:51:28,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10144fc6ca6000d, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:28,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:28,046 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0450ab8807f5,45065,1731761487895 2024-11-16T12:51:28,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45447-0x10144fc6ca6000f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:28,051 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/hbase.id] with ID: 40e21811-f531-400a-9fda-17d4ced55da8 2024-11-16T12:51:28,051 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/.tmp/hbase.id 2024-11-16T12:51:28,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741863_1041 (size=42) 2024-11-16T12:51:28,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741863_1041 (size=42) 2024-11-16T12:51:28,060 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/.tmp/hbase.id]:[hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/hbase.id] 2024-11-16T12:51:28,075 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:28,076 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T12:51:28,077 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-16T12:51:28,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:28,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10144fc6ca6000d, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:28,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43297-0x10144fc6ca6000e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:28,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45447-0x10144fc6ca6000f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:28,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741864_1042 (size=196) 2024-11-16T12:51:28,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741864_1042 (size=196) 2024-11-16T12:51:28,122 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T12:51:28,122 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T12:51:28,123 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:51:28,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741865_1043 (size=1189) 2024-11-16T12:51:28,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741865_1043 (size=1189) 2024-11-16T12:51:28,133 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store 2024-11-16T12:51:28,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741866_1044 (size=34) 2024-11-16T12:51:28,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741866_1044 (size=34) 2024-11-16T12:51:28,141 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:28,141 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T12:51:28,141 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:28,141 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:28,141 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T12:51:28,141 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:28,141 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:28,141 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731761488141Disabling compacts and flushes for region at 1731761488141Disabling writes for close at 1731761488141Writing region close event to WAL at 1731761488141Closed at 1731761488141 2024-11-16T12:51:28,142 WARN [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/.initializing 2024-11-16T12:51:28,142 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,45065,1731761487895 2024-11-16T12:51:28,144 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T12:51:28,145 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0450ab8807f5%2C45065%2C1731761487895, suffix=, logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,45065,1731761487895, archiveDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/oldWALs, maxLogs=10 2024-11-16T12:51:28,158 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,45065,1731761487895/0450ab8807f5%2C45065%2C1731761487895.1731761488146, exclude list is [], retry=0 2024-11-16T12:51:28,161 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43257,DS-06d91055-02a4-40d7-bddb-15aee81f02ac,DISK] 2024-11-16T12:51:28,161 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36961,DS-7ef9e9a6-250e-467c-989b-a9ffcab2299e,DISK] 2024-11-16T12:51:28,164 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,45065,1731761487895/0450ab8807f5%2C45065%2C1731761487895.1731761488146 2024-11-16T12:51:28,164 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:46667:46667)] 2024-11-16T12:51:28,164 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-16T12:51:28,165 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:28,165 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:28,165 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:28,167 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:28,168 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-16T12:51:28,169 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:28,169 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:28,169 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:28,171 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-16T12:51:28,171 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:28,171 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:28,172 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:28,173 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-16T12:51:28,173 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:28,174 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:28,174 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:28,176 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-16T12:51:28,176 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:28,177 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:28,177 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:28,177 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:28,178 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:28,179 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:28,179 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:28,180 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T12:51:28,181 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:28,183 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:51:28,184 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75341001, jitterRate=0.12266840040683746}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T12:51:28,185 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731761488165Initializing all the Stores at 1731761488166 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761488166Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761488166Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761488166Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761488167 (+1 ms)Cleaning up temporary data from old regions at 1731761488179 (+12 ms)Region opened successfully at 1731761488185 (+6 ms) 2024-11-16T12:51:28,185 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-16T12:51:28,188 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a633581, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0450ab8807f5/172.17.0.2:0 2024-11-16T12:51:28,189 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-16T12:51:28,189 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-16T12:51:28,189 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-16T12:51:28,190 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T12:51:28,190 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T12:51:28,190 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-16T12:51:28,190 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-16T12:51:28,193 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-16T12:51:28,194 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45065-0x10144fc6ca6000d, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-16T12:51:28,212 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-16T12:51:28,213 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-16T12:51:28,213 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45065-0x10144fc6ca6000d, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-16T12:51:28,221 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-16T12:51:28,221 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-16T12:51:28,222 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45065-0x10144fc6ca6000d, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-16T12:51:28,229 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-16T12:51:28,230 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45065-0x10144fc6ca6000d, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-16T12:51:28,237 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-16T12:51:28,240 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45065-0x10144fc6ca6000d, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-16T12:51:28,245 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-16T12:51:28,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45447-0x10144fc6ca6000f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T12:51:28,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T12:51:28,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43297-0x10144fc6ca6000e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T12:51:28,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10144fc6ca6000d, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T12:51:28,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10144fc6ca6000d, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:28,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45447-0x10144fc6ca6000f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:28,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43297-0x10144fc6ca6000e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:28,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:28,255 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0450ab8807f5,45065,1731761487895, sessionid=0x10144fc6ca6000d, setting cluster-up flag (Was=false) 2024-11-16T12:51:28,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45447-0x10144fc6ca6000f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:28,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10144fc6ca6000d, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:28,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43297-0x10144fc6ca6000e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:28,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:28,296 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-16T12:51:28,297 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0450ab8807f5,45065,1731761487895 2024-11-16T12:51:28,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10144fc6ca6000d, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:28,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:28,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45447-0x10144fc6ca6000f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:28,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43297-0x10144fc6ca6000e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:28,337 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-16T12:51:28,339 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0450ab8807f5,45065,1731761487895 2024-11-16T12:51:28,340 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-16T12:51:28,342 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T12:51:28,343 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-16T12:51:28,343 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0450ab8807f5,45065,1731761487895 Number of backup masters: 1 0450ab8807f5,34423,1731761487794 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-16T12:51:28,344 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:28,345 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:28,345 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:28,345 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:28,345 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0450ab8807f5:0, corePoolSize=10, maxPoolSize=10 2024-11-16T12:51:28,345 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:28,345 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0450ab8807f5:0, corePoolSize=2, maxPoolSize=2 2024-11-16T12:51:28,345 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:28,346 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731761518346 2024-11-16T12:51:28,346 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-16T12:51:28,346 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-16T12:51:28,346 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-16T12:51:28,346 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-16T12:51:28,346 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-16T12:51:28,347 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-16T12:51:28,347 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:28,347 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-16T12:51:28,347 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T12:51:28,347 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-16T12:51:28,347 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-16T12:51:28,347 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-16T12:51:28,348 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:28,349 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-16T12:51:28,350 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-16T12:51:28,350 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-16T12:51:28,353 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.large.0-1731761488350,5,FailOnTimeoutGroup] 2024-11-16T12:51:28,353 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.small.0-1731761488353,5,FailOnTimeoutGroup] 2024-11-16T12:51:28,353 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:28,354 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-16T12:51:28,354 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:28,354 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:28,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741868_1046 (size=1321) 2024-11-16T12:51:28,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741868_1046 (size=1321) 2024-11-16T12:51:28,362 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-16T12:51:28,362 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:28,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741869_1047 (size=32) 2024-11-16T12:51:28,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741869_1047 (size=32) 2024-11-16T12:51:28,373 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:28,374 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T12:51:28,376 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T12:51:28,376 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:28,377 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:28,378 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T12:51:28,379 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T12:51:28,379 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:28,380 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:28,380 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T12:51:28,381 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T12:51:28,381 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:28,382 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:28,382 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T12:51:28,383 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T12:51:28,384 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:28,384 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:28,384 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T12:51:28,385 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740 2024-11-16T12:51:28,385 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740 2024-11-16T12:51:28,387 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T12:51:28,387 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T12:51:28,387 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T12:51:28,388 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T12:51:28,391 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:51:28,391 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63847865, jitterRate=-0.04859267175197601}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T12:51:28,392 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731761488373Initializing all the Stores at 1731761488374 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761488374Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761488374Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761488374Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761488374Cleaning up temporary data from old regions at 1731761488387 (+13 ms)Region opened successfully at 1731761488392 (+5 ms) 2024-11-16T12:51:28,392 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T12:51:28,392 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T12:51:28,392 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T12:51:28,392 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T12:51:28,392 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T12:51:28,393 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T12:51:28,393 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731761488392Disabling compacts and flushes for region at 1731761488392Disabling writes for close at 1731761488392Writing region close event to WAL at 1731761488392Closed at 1731761488392 2024-11-16T12:51:28,394 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T12:51:28,394 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-16T12:51:28,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T12:51:28,395 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T12:51:28,396 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-16T12:51:28,402 INFO [RS:0;0450ab8807f5:43297 {}] regionserver.HRegionServer(746): ClusterId : 40e21811-f531-400a-9fda-17d4ced55da8 2024-11-16T12:51:28,402 INFO [RS:1;0450ab8807f5:45447 {}] regionserver.HRegionServer(746): ClusterId : 40e21811-f531-400a-9fda-17d4ced55da8 2024-11-16T12:51:28,403 DEBUG [RS:0;0450ab8807f5:43297 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T12:51:28,403 DEBUG [RS:1;0450ab8807f5:45447 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T12:51:28,429 DEBUG [RS:1;0450ab8807f5:45447 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T12:51:28,429 DEBUG [RS:0;0450ab8807f5:43297 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T12:51:28,429 DEBUG [RS:0;0450ab8807f5:43297 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T12:51:28,429 DEBUG [RS:1;0450ab8807f5:45447 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T12:51:28,438 DEBUG [RS:1;0450ab8807f5:45447 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T12:51:28,438 DEBUG [RS:0;0450ab8807f5:43297 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T12:51:28,438 DEBUG [RS:1;0450ab8807f5:45447 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58f5cb72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0450ab8807f5/172.17.0.2:0 2024-11-16T12:51:28,438 DEBUG [RS:0;0450ab8807f5:43297 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1818e18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0450ab8807f5/172.17.0.2:0 2024-11-16T12:51:28,449 DEBUG [RS:0;0450ab8807f5:43297 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0450ab8807f5:43297 2024-11-16T12:51:28,449 DEBUG [RS:1;0450ab8807f5:45447 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;0450ab8807f5:45447 2024-11-16T12:51:28,449 INFO [RS:1;0450ab8807f5:45447 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T12:51:28,449 INFO [RS:1;0450ab8807f5:45447 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T12:51:28,449 INFO [RS:0;0450ab8807f5:43297 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T12:51:28,449 DEBUG [RS:1;0450ab8807f5:45447 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T12:51:28,449 INFO [RS:0;0450ab8807f5:43297 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T12:51:28,449 DEBUG [RS:0;0450ab8807f5:43297 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T12:51:28,450 INFO [RS:0;0450ab8807f5:43297 {}] regionserver.HRegionServer(2659): reportForDuty to master=0450ab8807f5,45065,1731761487895 with port=43297, startcode=1731761487945 2024-11-16T12:51:28,450 INFO [RS:1;0450ab8807f5:45447 {}] regionserver.HRegionServer(2659): reportForDuty to master=0450ab8807f5,45065,1731761487895 with port=45447, startcode=1731761487981 2024-11-16T12:51:28,450 DEBUG [RS:1;0450ab8807f5:45447 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T12:51:28,450 DEBUG [RS:0;0450ab8807f5:43297 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T12:51:28,452 INFO [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42519, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T12:51:28,452 INFO [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42215, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T12:51:28,452 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45065 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0450ab8807f5,43297,1731761487945 2024-11-16T12:51:28,452 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45065 {}] master.ServerManager(517): Registering regionserver=0450ab8807f5,43297,1731761487945 2024-11-16T12:51:28,454 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45065 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0450ab8807f5,45447,1731761487981 2024-11-16T12:51:28,454 DEBUG [RS:0;0450ab8807f5:43297 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:28,454 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45065 {}] master.ServerManager(517): Registering regionserver=0450ab8807f5,45447,1731761487981 2024-11-16T12:51:28,454 DEBUG [RS:0;0450ab8807f5:43297 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44975 2024-11-16T12:51:28,454 DEBUG [RS:0;0450ab8807f5:43297 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T12:51:28,456 DEBUG [RS:1;0450ab8807f5:45447 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:28,456 DEBUG [RS:1;0450ab8807f5:45447 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44975 2024-11-16T12:51:28,456 DEBUG [RS:1;0450ab8807f5:45447 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T12:51:28,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10144fc6ca6000d, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T12:51:28,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T12:51:28,471 DEBUG [RS:0;0450ab8807f5:43297 {}] zookeeper.ZKUtil(111): regionserver:43297-0x10144fc6ca6000e, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0450ab8807f5,43297,1731761487945 2024-11-16T12:51:28,471 WARN [RS:0;0450ab8807f5:43297 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T12:51:28,471 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0450ab8807f5,43297,1731761487945] 2024-11-16T12:51:28,471 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0450ab8807f5,45447,1731761487981] 2024-11-16T12:51:28,471 INFO [RS:0;0450ab8807f5:43297 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:51:28,471 DEBUG [RS:0;0450ab8807f5:43297 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,43297,1731761487945 2024-11-16T12:51:28,471 DEBUG [RS:1;0450ab8807f5:45447 {}] zookeeper.ZKUtil(111): regionserver:45447-0x10144fc6ca6000f, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0450ab8807f5,45447,1731761487981 2024-11-16T12:51:28,471 WARN [RS:1;0450ab8807f5:45447 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T12:51:28,471 INFO [RS:1;0450ab8807f5:45447 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:51:28,471 DEBUG [RS:1;0450ab8807f5:45447 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,45447,1731761487981 2024-11-16T12:51:28,475 INFO [RS:0;0450ab8807f5:43297 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T12:51:28,475 INFO [RS:1;0450ab8807f5:45447 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T12:51:28,477 INFO [RS:0;0450ab8807f5:43297 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T12:51:28,478 INFO [RS:0;0450ab8807f5:43297 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T12:51:28,478 INFO [RS:0;0450ab8807f5:43297 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:28,478 INFO [RS:0;0450ab8807f5:43297 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T12:51:28,479 INFO [RS:0;0450ab8807f5:43297 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T12:51:28,479 INFO [RS:0;0450ab8807f5:43297 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:28,479 DEBUG [RS:0;0450ab8807f5:43297 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:28,480 DEBUG [RS:0;0450ab8807f5:43297 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:28,480 DEBUG [RS:0;0450ab8807f5:43297 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:28,480 DEBUG [RS:0;0450ab8807f5:43297 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:28,480 DEBUG [RS:0;0450ab8807f5:43297 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:28,480 DEBUG [RS:0;0450ab8807f5:43297 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0, corePoolSize=2, maxPoolSize=2 2024-11-16T12:51:28,480 DEBUG [RS:0;0450ab8807f5:43297 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:28,480 DEBUG [RS:0;0450ab8807f5:43297 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:28,480 DEBUG [RS:0;0450ab8807f5:43297 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:28,480 DEBUG [RS:0;0450ab8807f5:43297 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:28,480 DEBUG [RS:0;0450ab8807f5:43297 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:28,480 INFO [RS:1;0450ab8807f5:45447 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T12:51:28,480 DEBUG [RS:0;0450ab8807f5:43297 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:28,480 DEBUG [RS:0;0450ab8807f5:43297 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:51:28,481 DEBUG [RS:0;0450ab8807f5:43297 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:51:28,481 INFO [RS:1;0450ab8807f5:45447 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T12:51:28,481 INFO [RS:1;0450ab8807f5:45447 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:28,481 INFO [RS:1;0450ab8807f5:45447 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T12:51:28,481 INFO [RS:0;0450ab8807f5:43297 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:28,481 INFO [RS:0;0450ab8807f5:43297 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:28,481 INFO [RS:0;0450ab8807f5:43297 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:28,481 INFO [RS:0;0450ab8807f5:43297 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:28,482 INFO [RS:0;0450ab8807f5:43297 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:28,482 INFO [RS:0;0450ab8807f5:43297 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,43297,1731761487945-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T12:51:28,482 INFO [RS:1;0450ab8807f5:45447 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T12:51:28,482 INFO [RS:1;0450ab8807f5:45447 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:28,482 DEBUG [RS:1;0450ab8807f5:45447 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:28,482 DEBUG [RS:1;0450ab8807f5:45447 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:28,482 DEBUG [RS:1;0450ab8807f5:45447 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:28,483 DEBUG [RS:1;0450ab8807f5:45447 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:28,483 DEBUG [RS:1;0450ab8807f5:45447 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:28,483 DEBUG [RS:1;0450ab8807f5:45447 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0, corePoolSize=2, maxPoolSize=2 2024-11-16T12:51:28,483 DEBUG [RS:1;0450ab8807f5:45447 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:28,483 DEBUG [RS:1;0450ab8807f5:45447 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:28,483 DEBUG [RS:1;0450ab8807f5:45447 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:28,483 DEBUG [RS:1;0450ab8807f5:45447 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:28,483 DEBUG [RS:1;0450ab8807f5:45447 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:28,483 DEBUG [RS:1;0450ab8807f5:45447 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:28,483 DEBUG [RS:1;0450ab8807f5:45447 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:51:28,483 DEBUG [RS:1;0450ab8807f5:45447 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:51:28,485 INFO [RS:1;0450ab8807f5:45447 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:28,485 INFO [RS:1;0450ab8807f5:45447 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:28,485 INFO [RS:1;0450ab8807f5:45447 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:28,486 INFO [RS:1;0450ab8807f5:45447 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:28,486 INFO [RS:1;0450ab8807f5:45447 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:28,486 INFO [RS:1;0450ab8807f5:45447 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,45447,1731761487981-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T12:51:28,499 INFO [RS:1;0450ab8807f5:45447 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T12:51:28,499 INFO [RS:1;0450ab8807f5:45447 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,45447,1731761487981-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:28,499 INFO [RS:1;0450ab8807f5:45447 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:28,500 INFO [RS:1;0450ab8807f5:45447 {}] regionserver.Replication(171): 0450ab8807f5,45447,1731761487981 started 2024-11-16T12:51:28,501 INFO [RS:0;0450ab8807f5:43297 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T12:51:28,501 INFO [RS:0;0450ab8807f5:43297 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,43297,1731761487945-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:28,501 INFO [RS:0;0450ab8807f5:43297 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:28,501 INFO [RS:0;0450ab8807f5:43297 {}] regionserver.Replication(171): 0450ab8807f5,43297,1731761487945 started 2024-11-16T12:51:28,512 INFO [RS:1;0450ab8807f5:45447 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:28,513 INFO [RS:1;0450ab8807f5:45447 {}] regionserver.HRegionServer(1482): Serving as 0450ab8807f5,45447,1731761487981, RpcServer on 0450ab8807f5/172.17.0.2:45447, sessionid=0x10144fc6ca6000f 2024-11-16T12:51:28,513 DEBUG [RS:1;0450ab8807f5:45447 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T12:51:28,513 DEBUG [RS:1;0450ab8807f5:45447 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0450ab8807f5,45447,1731761487981 2024-11-16T12:51:28,513 DEBUG [RS:1;0450ab8807f5:45447 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,45447,1731761487981' 2024-11-16T12:51:28,513 DEBUG [RS:1;0450ab8807f5:45447 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T12:51:28,513 DEBUG [RS:1;0450ab8807f5:45447 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T12:51:28,514 DEBUG [RS:1;0450ab8807f5:45447 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T12:51:28,514 DEBUG [RS:1;0450ab8807f5:45447 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T12:51:28,514 DEBUG [RS:1;0450ab8807f5:45447 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0450ab8807f5,45447,1731761487981 2024-11-16T12:51:28,514 DEBUG [RS:1;0450ab8807f5:45447 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,45447,1731761487981' 2024-11-16T12:51:28,514 DEBUG [RS:1;0450ab8807f5:45447 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T12:51:28,514 DEBUG [RS:1;0450ab8807f5:45447 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T12:51:28,515 DEBUG [RS:1;0450ab8807f5:45447 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T12:51:28,515 INFO [RS:1;0450ab8807f5:45447 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T12:51:28,515 INFO [RS:1;0450ab8807f5:45447 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T12:51:28,520 INFO [RS:0;0450ab8807f5:43297 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:28,520 INFO [RS:0;0450ab8807f5:43297 {}] regionserver.HRegionServer(1482): Serving as 0450ab8807f5,43297,1731761487945, RpcServer on 0450ab8807f5/172.17.0.2:43297, sessionid=0x10144fc6ca6000e 2024-11-16T12:51:28,520 DEBUG [RS:0;0450ab8807f5:43297 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T12:51:28,520 DEBUG [RS:0;0450ab8807f5:43297 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0450ab8807f5,43297,1731761487945 2024-11-16T12:51:28,520 DEBUG [RS:0;0450ab8807f5:43297 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,43297,1731761487945' 2024-11-16T12:51:28,520 DEBUG [RS:0;0450ab8807f5:43297 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T12:51:28,521 DEBUG [RS:0;0450ab8807f5:43297 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T12:51:28,521 DEBUG [RS:0;0450ab8807f5:43297 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T12:51:28,521 DEBUG [RS:0;0450ab8807f5:43297 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T12:51:28,521 DEBUG [RS:0;0450ab8807f5:43297 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0450ab8807f5,43297,1731761487945 2024-11-16T12:51:28,521 DEBUG [RS:0;0450ab8807f5:43297 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,43297,1731761487945' 2024-11-16T12:51:28,521 DEBUG [RS:0;0450ab8807f5:43297 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T12:51:28,522 DEBUG [RS:0;0450ab8807f5:43297 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T12:51:28,522 DEBUG [RS:0;0450ab8807f5:43297 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T12:51:28,522 INFO [RS:0;0450ab8807f5:43297 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T12:51:28,522 INFO [RS:0;0450ab8807f5:43297 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T12:51:28,547 WARN [0450ab8807f5:45065 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-16T12:51:28,615 INFO [RS:1;0450ab8807f5:45447 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T12:51:28,617 INFO [RS:1;0450ab8807f5:45447 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0450ab8807f5%2C45447%2C1731761487981, suffix=, logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,45447,1731761487981, archiveDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/oldWALs, maxLogs=32 2024-11-16T12:51:28,623 INFO [RS:0;0450ab8807f5:43297 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T12:51:28,625 INFO [RS:0;0450ab8807f5:43297 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0450ab8807f5%2C43297%2C1731761487945, suffix=, logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,43297,1731761487945, archiveDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/oldWALs, maxLogs=32 2024-11-16T12:51:28,631 DEBUG [RS:1;0450ab8807f5:45447 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,45447,1731761487981/0450ab8807f5%2C45447%2C1731761487981.1731761488618, exclude list is [], retry=0 2024-11-16T12:51:28,634 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36961,DS-7ef9e9a6-250e-467c-989b-a9ffcab2299e,DISK] 2024-11-16T12:51:28,634 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43257,DS-06d91055-02a4-40d7-bddb-15aee81f02ac,DISK] 2024-11-16T12:51:28,637 INFO [RS:1;0450ab8807f5:45447 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,45447,1731761487981/0450ab8807f5%2C45447%2C1731761487981.1731761488618 2024-11-16T12:51:28,637 DEBUG [RS:1;0450ab8807f5:45447 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46667:46667),(127.0.0.1/127.0.0.1:38819:38819)] 2024-11-16T12:51:28,644 DEBUG [RS:0;0450ab8807f5:43297 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,43297,1731761487945/0450ab8807f5%2C43297%2C1731761487945.1731761488626, exclude list is [], retry=0 2024-11-16T12:51:28,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43257,DS-06d91055-02a4-40d7-bddb-15aee81f02ac,DISK] 2024-11-16T12:51:28,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36961,DS-7ef9e9a6-250e-467c-989b-a9ffcab2299e,DISK] 2024-11-16T12:51:28,650 INFO [RS:0;0450ab8807f5:43297 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,43297,1731761487945/0450ab8807f5%2C43297%2C1731761487945.1731761488626 2024-11-16T12:51:28,651 DEBUG [RS:0;0450ab8807f5:43297 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46667:46667),(127.0.0.1/127.0.0.1:38819:38819)] 2024-11-16T12:51:28,797 DEBUG [0450ab8807f5:45065 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-16T12:51:28,797 DEBUG [0450ab8807f5:45065 {}] balancer.BalancerClusterState(204): Hosts are {0450ab8807f5=0} racks are {/default-rack=0} 2024-11-16T12:51:28,799 DEBUG [0450ab8807f5:45065 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T12:51:28,799 DEBUG [0450ab8807f5:45065 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T12:51:28,799 DEBUG [0450ab8807f5:45065 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T12:51:28,799 DEBUG [0450ab8807f5:45065 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T12:51:28,800 INFO [0450ab8807f5:45065 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T12:51:28,800 INFO [0450ab8807f5:45065 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T12:51:28,800 DEBUG [0450ab8807f5:45065 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T12:51:28,800 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0450ab8807f5,45447,1731761487981 2024-11-16T12:51:28,801 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0450ab8807f5,45447,1731761487981, state=OPENING 2024-11-16T12:51:28,829 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-16T12:51:28,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43297-0x10144fc6ca6000e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:28,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10144fc6ca6000d, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:28,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:28,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45447-0x10144fc6ca6000f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:28,838 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T12:51:28,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0450ab8807f5,45447,1731761487981}] 2024-11-16T12:51:28,838 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:28,838 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:28,838 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:28,838 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:28,991 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T12:51:28,993 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52007, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T12:51:28,996 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T12:51:28,997 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:51:28,997 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T12:51:28,999 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0450ab8807f5%2C45447%2C1731761487981.meta, suffix=.meta, logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,45447,1731761487981, archiveDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/oldWALs, maxLogs=32 2024-11-16T12:51:29,011 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,45447,1731761487981/0450ab8807f5%2C45447%2C1731761487981.meta.1731761488999.meta, exclude list is [], retry=0 2024-11-16T12:51:29,014 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36961,DS-7ef9e9a6-250e-467c-989b-a9ffcab2299e,DISK] 2024-11-16T12:51:29,014 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43257,DS-06d91055-02a4-40d7-bddb-15aee81f02ac,DISK] 2024-11-16T12:51:29,016 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,45447,1731761487981/0450ab8807f5%2C45447%2C1731761487981.meta.1731761488999.meta 2024-11-16T12:51:29,017 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46667:46667),(127.0.0.1/127.0.0.1:38819:38819)] 2024-11-16T12:51:29,017 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T12:51:29,017 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T12:51:29,017 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T12:51:29,018 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T12:51:29,018 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T12:51:29,018 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:29,018 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T12:51:29,018 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T12:51:29,020 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T12:51:29,021 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T12:51:29,021 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:29,021 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:29,021 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T12:51:29,022 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T12:51:29,022 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:29,023 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:29,023 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T12:51:29,024 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T12:51:29,024 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:29,024 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:29,024 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T12:51:29,025 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T12:51:29,025 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:29,025 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:29,026 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T12:51:29,026 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740 2024-11-16T12:51:29,028 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740 2024-11-16T12:51:29,029 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T12:51:29,030 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T12:51:29,030 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T12:51:29,031 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T12:51:29,032 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64399467, jitterRate=-0.04037316143512726}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T12:51:29,032 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T12:51:29,033 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731761489018Writing region info on filesystem at 1731761489018Initializing all the Stores at 1731761489019 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761489019Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761489020 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761489020Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761489020Cleaning up temporary data from old regions at 1731761489030 (+10 ms)Running coprocessor post-open hooks at 1731761489032 (+2 ms)Region opened successfully at 1731761489033 (+1 ms) 2024-11-16T12:51:29,034 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731761488991 2024-11-16T12:51:29,036 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T12:51:29,036 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T12:51:29,037 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0450ab8807f5,45447,1731761487981 2024-11-16T12:51:29,038 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0450ab8807f5,45447,1731761487981, state=OPEN 2024-11-16T12:51:29,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45447-0x10144fc6ca6000f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:29,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43297-0x10144fc6ca6000e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:29,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:29,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10144fc6ca6000d, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:29,071 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0450ab8807f5,45447,1731761487981 2024-11-16T12:51:29,071 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:29,071 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:29,071 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:29,071 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:29,074 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-16T12:51:29,075 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0450ab8807f5,45447,1731761487981 in 233 msec 2024-11-16T12:51:29,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-16T12:51:29,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 680 msec 2024-11-16T12:51:29,078 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T12:51:29,078 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-16T12:51:29,079 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T12:51:29,079 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,45447,1731761487981, seqNum=-1] 2024-11-16T12:51:29,080 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T12:51:29,081 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60329, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T12:51:29,087 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 744 msec 2024-11-16T12:51:29,087 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731761489087, completionTime=-1 2024-11-16T12:51:29,087 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-16T12:51:29,088 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T12:51:29,089 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T12:51:29,090 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731761549090 2024-11-16T12:51:29,090 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731761609090 2024-11-16T12:51:29,090 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-16T12:51:29,090 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-16T12:51:29,090 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,45065,1731761487895-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:29,090 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,45065,1731761487895-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:29,091 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,45065,1731761487895-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:29,091 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0450ab8807f5:45065, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:29,091 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:29,091 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:29,094 DEBUG [master/0450ab8807f5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-16T12:51:29,095 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.049sec 2024-11-16T12:51:29,096 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-16T12:51:29,096 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-16T12:51:29,096 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-16T12:51:29,096 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-16T12:51:29,096 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-16T12:51:29,096 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,45065,1731761487895-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T12:51:29,096 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,45065,1731761487895-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-16T12:51:29,098 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-16T12:51:29,098 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-16T12:51:29,098 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,45065,1731761487895-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:29,103 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2d714c78 to 127.0.0.1:53208 with session timeout=1000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@14514ece 2024-11-16T12:51:29,113 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6939091d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T12:51:29,114 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T12:51:29,115 DEBUG [ReadOnlyZKClient-127.0.0.1:53208@0x2d714c78 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,45447,1731761487981, seqNum=-1] 2024-11-16T12:51:29,115 DEBUG [ReadOnlyZKClient-127.0.0.1:53208@0x2d714c78 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T12:51:29,117 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36480, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T12:51:29,119 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0450ab8807f5,45065,1731761487895 2024-11-16T12:51:29,119 INFO [Time-limited test {}] hbase.TestZooKeeper(138): Starting testMasterZKSessionRecoveryFailure 2024-11-16T12:51:29,119 ERROR [Time-limited test {}] master.HMaster(3218): Master server abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-16T12:51:29,119 ERROR [Time-limited test {}] master.HMaster(3223): ***** ABORTING master 0450ab8807f5,45065,1731761487895: Test recovery from zk session expired ***** org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired at org.apache.hadoop.hbase.TestZooKeeper.testMasterZKSessionRecoveryFailure(TestZooKeeper.java:141) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:29,119 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master '0450ab8807f5,45065,1731761487895' ***** 2024-11-16T12:51:29,119 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: Stopped by Time-limited test 2024-11-16T12:51:29,119 INFO [M:1;0450ab8807f5:45065 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T12:51:29,119 INFO [M:1;0450ab8807f5:45065 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:1;0450ab8807f5:45065. 2024-11-16T12:51:29,119 DEBUG [M:1;0450ab8807f5:45065 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:51:29,119 DEBUG [M:1;0450ab8807f5:45065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:29,120 INFO [M:1;0450ab8807f5:45065 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T12:51:29,120 DEBUG [M:1;0450ab8807f5:45065 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T12:51:29,120 DEBUG [M:1;0450ab8807f5:45065 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T12:51:29,120 DEBUG [master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.large.0-1731761488350 {}] cleaner.HFileCleaner(306): Exit Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.large.0-1731761488350,5,FailOnTimeoutGroup] 2024-11-16T12:51:29,120 DEBUG [master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.small.0-1731761488353 {}] cleaner.HFileCleaner(306): Exit Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.small.0-1731761488353,5,FailOnTimeoutGroup] 2024-11-16T12:51:29,120 INFO [M:1;0450ab8807f5:45065 {}] hbase.ChoreService(370): Chore service for: master/0450ab8807f5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T12:51:29,120 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T12:51:29,120 INFO [M:1;0450ab8807f5:45065 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T12:51:29,120 DEBUG [M:1;0450ab8807f5:45065 {}] master.HMaster(1795): Stopping service threads 2024-11-16T12:51:29,120 INFO [M:1;0450ab8807f5:45065 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T12:51:29,120 INFO [M:1;0450ab8807f5:45065 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T12:51:29,121 INFO [M:1;0450ab8807f5:45065 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=true 2024-11-16T12:51:29,121 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T12:51:29,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43297-0x10144fc6ca6000e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:29,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45447-0x10144fc6ca6000f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:29,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:29,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10144fc6ca6000d, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:29,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:29,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10144fc6ca6000d, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:29,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45447-0x10144fc6ca6000f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:29,129 DEBUG [M:1;0450ab8807f5:45065 {}] zookeeper.ZKUtil(347): master:45065-0x10144fc6ca6000d, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-16T12:51:29,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43297-0x10144fc6ca6000e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:29,129 WARN [M:1;0450ab8807f5:45065 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-16T12:51:29,129 INFO [Time-limited test {}] hbase.TestZooKeeper(153): Creating table testMasterZKSessionRecoveryFailure_1731761489129 2024-11-16T12:51:29,129 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T12:51:29,129 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:29,129 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45447-0x10144fc6ca6000f, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:29,130 DEBUG [zk-event-processor-pool-0 {}] master.ActiveMasterManager(204): No master available. Notifying waiting threads 2024-11-16T12:51:29,130 INFO [M:1;0450ab8807f5:45065 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/.lastflushedseqids 2024-11-16T12:51:29,130 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43297-0x10144fc6ca6000e, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:29,130 DEBUG [ReadOnlyZKClient-127.0.0.1:53208@0x2d714c78 {}] client.ConnectionUtils(551): Failed to fetch master stub from registry org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase/master at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:29,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741873_1051 (size=108) 2024-11-16T12:51:29,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741873_1051 (size=108) 2024-11-16T12:51:29,137 INFO [M:1;0450ab8807f5:45065 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T12:51:29,137 INFO [M:1;0450ab8807f5:45065 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=true 2024-11-16T12:51:29,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43297-0x10144fc6ca6000e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:29,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:29,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10144fc6ca6000d, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:29,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45447-0x10144fc6ca6000f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:29,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45447-0x10144fc6ca6000f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:29,138 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T12:51:29,138 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T12:51:29,138 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0450ab8807f5,34423,1731761487794 from backup master directory 2024-11-16T12:51:29,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741867_1045 (size=9664) 2024-11-16T12:51:29,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741867_1045 (size=9664) 2024-11-16T12:51:29,140 DEBUG [M:1;0450ab8807f5:45065 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T12:51:29,140 INFO [M:1;0450ab8807f5:45065 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:29,141 DEBUG [M:1;0450ab8807f5:45065 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:29,141 DEBUG [M:1;0450ab8807f5:45065 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T12:51:29,141 DEBUG [M:1;0450ab8807f5:45065 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:29,141 ERROR [M:1;0450ab8807f5:45065 {}] regionserver.HRegion(1960): Memstore data size is 7867 in region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:29,141 INFO [M:1;0450ab8807f5:45065 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:29,141 DEBUG [M:1;0450ab8807f5:45065 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731761489140Disabling compacts and flushes for region at 1731761489140Disabling writes for close at 1731761489141 (+1 ms)Writing region close event to WAL at 1731761489141Closed at 1731761489141 2024-11-16T12:51:29,141 INFO [M:1;0450ab8807f5:45065 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-16T12:51:29,141 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T12:51:29,141 INFO [M:1;0450ab8807f5:45065 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45065 2024-11-16T12:51:29,142 INFO [M:1;0450ab8807f5:45065 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T12:51:29,146 WARN [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T12:51:29,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0450ab8807f5,34423,1731761487794 2024-11-16T12:51:29,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10144fc6ca6000d, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:29,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43297-0x10144fc6ca6000e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:29,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45447-0x10144fc6ca6000f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:29,146 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0450ab8807f5,34423,1731761487794 2024-11-16T12:51:29,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:29,147 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-14-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 0450ab8807f5:45065 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0450ab8807f5/172.17.0.2:45065 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:29,147 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-14-2 {}] ipc.FailedServers(52): Added failed server with address 0450ab8807f5:45065 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0450ab8807f5/172.17.0.2:45065 2024-11-16T12:51:29,160 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-14-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 0450ab8807f5:45065 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0450ab8807f5/172.17.0.2:45065 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:29,160 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-14-1 {}] ipc.FailedServers(52): Added failed server with address 0450ab8807f5:45065 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0450ab8807f5/172.17.0.2:45065 2024-11-16T12:51:29,161 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:29,161 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T12:51:29,166 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=1) cost 5ms. 2024-11-16T12:51:29,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741874_1052 (size=196) 2024-11-16T12:51:29,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741874_1052 (size=196) 2024-11-16T12:51:29,238 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T12:51:29,239 DEBUG [ReadOnlyZKClient-127.0.0.1:53208@0x2d714c78 {}] client.AsyncConnectionImpl(321): The fetched master address is 0450ab8807f5,34423,1731761487794 2024-11-16T12:51:29,239 DEBUG [ReadOnlyZKClient-127.0.0.1:53208@0x2d714c78 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7325204d 2024-11-16T12:51:29,240 DEBUG [ReadOnlyZKClient-127.0.0.1:53208@0x2d714c78 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T12:51:29,242 INFO [HMaster-EventLoopGroup-11-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T12:51:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34423 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.checkInitialized(HMaster.java:3340) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.createTable(HMaster.java:2437) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.createTable(MasterRpcServices.java:786) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:29,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10144fc6ca6000d, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:29,254 INFO [M:1;0450ab8807f5:45065 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T12:51:29,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10144fc6ca6000d, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:29,255 DEBUG [RS:1;0450ab8807f5:45447 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T12:51:29,256 INFO [HMaster-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44465, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T12:51:29,257 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34423 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:29,261 DEBUG [RS:0;0450ab8807f5:43297 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T12:51:29,263 INFO [HMaster-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40025, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T12:51:29,263 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34423 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:29,360 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34423 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:29,366 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34423 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34423 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.checkInitialized(HMaster.java:3340) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.createTable(HMaster.java:2437) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.createTable(MasterRpcServices.java:786) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:29,462 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34423 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:29,468 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34423 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:29,565 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34423 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:29,570 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34423 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:29,587 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T12:51:29,587 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T12:51:29,588 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:51:29,597 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(342): old store file tracker DEFAULT is the same with new store file tracker, skip migration 2024-11-16T12:51:29,600 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(316): Renamed hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,45065,1731761487895 to hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,45065,1731761487895-dead as it is dead 2024-11-16T12:51:29,601 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,45065,1731761487895-dead/0450ab8807f5%2C45065%2C1731761487895.1731761488146 2024-11-16T12:51:29,601 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,45065,1731761487895-dead/0450ab8807f5%2C45065%2C1731761487895.1731761488146 after 0ms 2024-11-16T12:51:29,602 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(328): Renamed hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,45065,1731761487895-dead/0450ab8807f5%2C45065%2C1731761487895.1731761488146 to hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.wals/0450ab8807f5%2C45065%2C1731761487895.1731761488146 2024-11-16T12:51:29,602 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(330): Delete empty local region wal dir hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,45065,1731761487895-dead 2024-11-16T12:51:29,602 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,34423,1731761487794 2024-11-16T12:51:29,603 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T12:51:29,605 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0450ab8807f5%2C34423%2C1731761487794, suffix=, logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,34423,1731761487794, archiveDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/oldWALs, maxLogs=10 2024-11-16T12:51:29,618 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,34423,1731761487794/0450ab8807f5%2C34423%2C1731761487794.1731761489605, exclude list is [], retry=0 2024-11-16T12:51:29,621 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36961,DS-7ef9e9a6-250e-467c-989b-a9ffcab2299e,DISK] 2024-11-16T12:51:29,621 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43257,DS-06d91055-02a4-40d7-bddb-15aee81f02ac,DISK] 2024-11-16T12:51:29,623 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,34423,1731761487794/0450ab8807f5%2C34423%2C1731761487794.1731761489605 2024-11-16T12:51:29,624 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46667:46667),(127.0.0.1/127.0.0.1:38819:38819)] 2024-11-16T12:51:29,624 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-16T12:51:29,624 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:29,624 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:29,624 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:29,626 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:29,627 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-16T12:51:29,627 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:29,627 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:29,628 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:29,628 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-16T12:51:29,628 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:29,629 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:29,629 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:29,630 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-16T12:51:29,630 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:29,630 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:29,631 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:29,631 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-16T12:51:29,631 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:29,632 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:29,632 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:29,633 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(5516): Found 1 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.wals 2024-11-16T12:51:29,633 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.wals/0450ab8807f5%2C45065%2C1731761487895.1731761488146 2024-11-16T12:51:29,640 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(5793): Applied 36, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=28, path=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.wals/0450ab8807f5%2C45065%2C1731761487895.1731761488146 2024-11-16T12:51:29,641 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.68 KB heapSize=11.36 KB 2024-11-16T12:51:29,660 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e58f389a89c04689ac8977068bda7da6 is 82, key is hbase:meta,,1/info:regioninfo/1731761489037/Put/seqid=0 2024-11-16T12:51:29,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741876_1054 (size=5672) 2024-11-16T12:51:29,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741876_1054 (size=5672) 2024-11-16T12:51:29,667 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e58f389a89c04689ac8977068bda7da6 2024-11-16T12:51:29,668 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34423 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:29,672 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34423 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:29,695 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7c0a361ccd7947749a01f37b32c5f7ba is 366, key is \x00\x00\x00\x00\x00\x00\x00\x02/proc:d/1731761489075/Put/seqid=0 2024-11-16T12:51:29,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741877_1055 (size=12229) 2024-11-16T12:51:29,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741877_1055 (size=12229) 2024-11-16T12:51:29,701 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7c0a361ccd7947749a01f37b32c5f7ba 2024-11-16T12:51:29,727 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/313a3ce38e1f4653b6a4b7c0459906ba is 69, key is 0450ab8807f5,43297,1731761487945/rs:state/1731761488453/Put/seqid=0 2024-11-16T12:51:29,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741878_1056 (size=5224) 2024-11-16T12:51:29,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741878_1056 (size=5224) 2024-11-16T12:51:29,733 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/313a3ce38e1f4653b6a4b7c0459906ba 2024-11-16T12:51:29,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34423 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.checkInitialized(HMaster.java:3340) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.createTable(HMaster.java:2437) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.createTable(MasterRpcServices.java:786) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:29,770 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34423 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:29,775 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34423 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:29,775 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e58f389a89c04689ac8977068bda7da6 as hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e58f389a89c04689ac8977068bda7da6 2024-11-16T12:51:29,785 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e58f389a89c04689ac8977068bda7da6, entries=8, sequenceid=28, filesize=5.5 K 2024-11-16T12:51:29,787 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7c0a361ccd7947749a01f37b32c5f7ba as hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7c0a361ccd7947749a01f37b32c5f7ba 2024-11-16T12:51:29,797 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7c0a361ccd7947749a01f37b32c5f7ba, entries=26, sequenceid=28, filesize=11.9 K 2024-11-16T12:51:29,799 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/313a3ce38e1f4653b6a4b7c0459906ba as hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/313a3ce38e1f4653b6a4b7c0459906ba 2024-11-16T12:51:29,809 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/313a3ce38e1f4653b6a4b7c0459906ba, entries=2, sequenceid=28, filesize=5.1 K 2024-11-16T12:51:29,810 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.68 KB/7867, heapSize ~11.06 KB/11328, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 170ms, sequenceid=28, compaction requested=false; wal=null 2024-11-16T12:51:29,811 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.wals/0450ab8807f5%2C45065%2C1731761487895.1731761488146 2024-11-16T12:51:29,813 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:29,813 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:29,814 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T12:51:29,816 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:29,824 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/28.seqid, newMaxSeqId=28, maxSeqId=1 2024-11-16T12:51:29,826 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=29; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61807734, jitterRate=-0.07899299263954163}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T12:51:29,827 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731761489624Initializing all the Stores at 1731761489626 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761489626Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761489626Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761489626Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761489626Obtaining lock to block concurrent updates at 1731761489641 (+15 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731761489641Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7867, getHeapSize=11568, getOffHeapSize=0, getCellsCount=36 at 1731761489641Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731761489641Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731761489641Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731761489659 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731761489660 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731761489675 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731761489694 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731761489694Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731761489709 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731761489727 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731761489727Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@728a83c9: reopening flushed file at 1731761489774 (+47 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75c96bb0: reopening flushed file at 1731761489785 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b6e1a20: reopening flushed file at 1731761489797 (+12 ms)Finished flush of dataSize ~7.68 KB/7867, heapSize ~11.06 KB/11328, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 170ms, sequenceid=28, compaction requested=false; wal=null at 1731761489810 (+13 ms)Cleaning up temporary data from old regions at 1731761489813 (+3 ms)Region opened successfully at 1731761489827 (+14 ms) 2024-11-16T12:51:29,829 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-16T12:51:29,835 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4caaeadc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0450ab8807f5/172.17.0.2:0 2024-11-16T12:51:29,838 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(884): The info family in master local region already has data in it, skip migrating... 2024-11-16T12:51:29,839 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-16T12:51:29,839 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-16T12:51:29,839 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T12:51:29,840 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T12:51:29,844 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T12:51:29,845 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-16T12:51:29,845 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-16T12:51:29,851 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=0450ab8807f5,45447,1731761487981, regionLocation=0450ab8807f5,45447,1731761487981, openSeqNum=2 2024-11-16T12:51:29,851 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=0450ab8807f5,45447,1731761487981, table=hbase:meta, region=1588230740 2024-11-16T12:51:29,851 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0450ab8807f5,45447,1731761487981, state=OPEN 2024-11-16T12:51:29,874 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34423 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:29,878 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34423 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:29,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:29,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45447-0x10144fc6ca6000f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:29,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43297-0x10144fc6ca6000e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:29,888 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:29,888 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:29,888 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:29,893 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 2 possibly 'live' servers, and 0 'splitting'. 2024-11-16T12:51:29,894 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ServerManager(517): Registering regionserver=0450ab8807f5,43297,1731761487945 2024-11-16T12:51:29,894 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ServerManager(517): Registering regionserver=0450ab8807f5,45447,1731761487981 2024-11-16T12:51:29,895 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-16T12:51:29,904 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-16T12:51:29,904 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-16T12:51:29,905 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-16T12:51:29,989 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34423 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:29,989 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34423 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:30,029 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-16T12:51:30,029 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-16T12:51:30,030 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-16T12:51:30,131 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34423 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:30,131 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34423 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:30,268 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T12:51:30,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34423 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.checkInitialized(HMaster.java:3340) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.createTable(HMaster.java:2437) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.createTable(MasterRpcServices.java:786) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:30,395 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-16T12:51:30,396 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-16T12:51:30,497 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34423 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:30,497 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34423 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:30,571 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-16T12:51:30,577 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-16T12:51:30,674 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34423 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:30,675 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34423 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:30,737 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-16T12:51:30,737 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0450ab8807f5,34423,1731761487794, sessionid=0x10144fc6ca6000c, setting cluster-up flag (Was=true) 2024-11-16T12:51:30,838 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34423 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:30,838 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34423 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:30,841 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-16T12:51:30,842 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0450ab8807f5,34423,1731761487794 2024-11-16T12:51:30,855 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-16T12:51:30,856 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0450ab8807f5,34423,1731761487794 2024-11-16T12:51:30,857 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ServerManager(1190): begin to load .lastflushedseqids at hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/.lastflushedseqids 2024-11-16T12:51:30,860 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-16T12:51:30,860 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0450ab8807f5,34423,1731761487794 Number of backup masters: 0 Number of live region servers: 2 0450ab8807f5,43297,1731761487945 0450ab8807f5,45447,1731761487981 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-16T12:51:30,861 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:30,861 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:30,861 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:30,861 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:30,862 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0450ab8807f5:0, corePoolSize=10, maxPoolSize=10 2024-11-16T12:51:30,862 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:30,862 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0450ab8807f5:0, corePoolSize=2, maxPoolSize=2 2024-11-16T12:51:30,862 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:30,865 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731761520864 2024-11-16T12:51:30,865 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-16T12:51:30,865 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-16T12:51:30,865 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-16T12:51:30,865 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-16T12:51:30,865 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-16T12:51:30,865 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-16T12:51:30,865 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:30,865 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-16T12:51:30,865 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-16T12:51:30,865 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-16T12:51:30,866 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-16T12:51:30,866 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-16T12:51:30,866 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.large.0-1731761490866,5,FailOnTimeoutGroup] 2024-11-16T12:51:30,866 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.small.0-1731761490866,5,FailOnTimeoutGroup] 2024-11-16T12:51:30,866 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:30,866 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-16T12:51:30,866 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:30,866 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:30,866 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731761490866, completionTime=-1 2024-11-16T12:51:30,866 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-16T12:51:30,866 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T12:51:30,867 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T12:51:30,867 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,45447,1731761487981, seqNum=-1] 2024-11-16T12:51:30,867 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T12:51:30,869 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37695, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T12:51:30,870 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T12:51:30,870 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731761550870 2024-11-16T12:51:30,870 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731761610870 2024-11-16T12:51:30,870 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-11-16T12:51:30,871 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,34423,1731761487794-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:30,871 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,34423,1731761487794-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:30,871 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,34423,1731761487794-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:30,871 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0450ab8807f5:34423, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:30,871 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:30,872 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:30,873 DEBUG [master/0450ab8807f5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-16T12:51:30,875 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.729sec 2024-11-16T12:51:30,876 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-16T12:51:30,876 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-16T12:51:30,876 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-16T12:51:30,876 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-16T12:51:30,876 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-16T12:51:30,876 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,34423,1731761487794-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T12:51:30,876 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,34423,1731761487794-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-16T12:51:30,878 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-16T12:51:30,878 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-16T12:51:30,878 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,34423,1731761487794-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:31,290 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34423 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testMasterZKSessionRecoveryFailure_1731761489129', {NAME => 'fam', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T12:51:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34423 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testMasterZKSessionRecoveryFailure_1731761489129 2024-11-16T12:51:31,293 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testMasterZKSessionRecoveryFailure_1731761489129 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T12:51:31,293 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:31,293 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34423 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testMasterZKSessionRecoveryFailure_1731761489129" procId is: 4 2024-11-16T12:51:31,295 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testMasterZKSessionRecoveryFailure_1731761489129 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T12:51:31,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T12:51:31,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741879_1057 (size=399) 2024-11-16T12:51:31,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741879_1057 (size=399) 2024-11-16T12:51:31,306 INFO [RegionOpenAndInit-testMasterZKSessionRecoveryFailure_1731761489129-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0413c710b53e90d189e19874b9658aed, NAME => 'testMasterZKSessionRecoveryFailure_1731761489129,,1731761491290.0413c710b53e90d189e19874b9658aed.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testMasterZKSessionRecoveryFailure_1731761489129', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'fam', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:31,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741880_1058 (size=83) 2024-11-16T12:51:31,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741880_1058 (size=83) 2024-11-16T12:51:31,319 DEBUG [RegionOpenAndInit-testMasterZKSessionRecoveryFailure_1731761489129-pool-0 {}] regionserver.HRegion(898): Instantiated testMasterZKSessionRecoveryFailure_1731761489129,,1731761491290.0413c710b53e90d189e19874b9658aed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:31,319 DEBUG [RegionOpenAndInit-testMasterZKSessionRecoveryFailure_1731761489129-pool-0 {}] regionserver.HRegion(1722): Closing 0413c710b53e90d189e19874b9658aed, disabling compactions & flushes 2024-11-16T12:51:31,319 INFO [RegionOpenAndInit-testMasterZKSessionRecoveryFailure_1731761489129-pool-0 {}] regionserver.HRegion(1755): Closing region testMasterZKSessionRecoveryFailure_1731761489129,,1731761491290.0413c710b53e90d189e19874b9658aed. 2024-11-16T12:51:31,319 DEBUG [RegionOpenAndInit-testMasterZKSessionRecoveryFailure_1731761489129-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testMasterZKSessionRecoveryFailure_1731761489129,,1731761491290.0413c710b53e90d189e19874b9658aed. 2024-11-16T12:51:31,319 DEBUG [RegionOpenAndInit-testMasterZKSessionRecoveryFailure_1731761489129-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testMasterZKSessionRecoveryFailure_1731761489129,,1731761491290.0413c710b53e90d189e19874b9658aed. after waiting 0 ms 2024-11-16T12:51:31,319 DEBUG [RegionOpenAndInit-testMasterZKSessionRecoveryFailure_1731761489129-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testMasterZKSessionRecoveryFailure_1731761489129,,1731761491290.0413c710b53e90d189e19874b9658aed. 2024-11-16T12:51:31,319 INFO [RegionOpenAndInit-testMasterZKSessionRecoveryFailure_1731761489129-pool-0 {}] regionserver.HRegion(1973): Closed testMasterZKSessionRecoveryFailure_1731761489129,,1731761491290.0413c710b53e90d189e19874b9658aed. 2024-11-16T12:51:31,319 DEBUG [RegionOpenAndInit-testMasterZKSessionRecoveryFailure_1731761489129-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0413c710b53e90d189e19874b9658aed: Waiting for close lock at 1731761491319Disabling compacts and flushes for region at 1731761491319Disabling writes for close at 1731761491319Writing region close event to WAL at 1731761491319Closed at 1731761491319 2024-11-16T12:51:31,321 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testMasterZKSessionRecoveryFailure_1731761489129 execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T12:51:31,322 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testMasterZKSessionRecoveryFailure_1731761489129,,1731761491290.0413c710b53e90d189e19874b9658aed.","families":{"info":[{"qualifier":"regioninfo","vlen":82,"tag":[],"timestamp":"1731761491321"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731761491321"}]},"ts":"1731761491321"} 2024-11-16T12:51:31,325 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T12:51:31,326 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testMasterZKSessionRecoveryFailure_1731761489129 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T12:51:31,327 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testMasterZKSessionRecoveryFailure_1731761489129","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731761491326"}]},"ts":"1731761491326"} 2024-11-16T12:51:31,329 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testMasterZKSessionRecoveryFailure_1731761489129, state=ENABLING in hbase:meta 2024-11-16T12:51:31,330 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {0450ab8807f5=0} racks are {/default-rack=0} 2024-11-16T12:51:31,330 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T12:51:31,330 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T12:51:31,330 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T12:51:31,330 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T12:51:31,330 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T12:51:31,330 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T12:51:31,331 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T12:51:31,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testMasterZKSessionRecoveryFailure_1731761489129, region=0413c710b53e90d189e19874b9658aed, ASSIGN}] 2024-11-16T12:51:31,332 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testMasterZKSessionRecoveryFailure_1731761489129, region=0413c710b53e90d189e19874b9658aed, ASSIGN 2024-11-16T12:51:31,333 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testMasterZKSessionRecoveryFailure_1731761489129, region=0413c710b53e90d189e19874b9658aed, ASSIGN; state=OFFLINE, location=0450ab8807f5,43297,1731761487945; forceNewPlan=false, retain=false 2024-11-16T12:51:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T12:51:31,484 INFO [0450ab8807f5:34423 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-16T12:51:31,484 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0413c710b53e90d189e19874b9658aed, regionState=OPENING, regionLocation=0450ab8807f5,43297,1731761487945 2024-11-16T12:51:31,487 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-14-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testMasterZKSessionRecoveryFailure_1731761489129, region=0413c710b53e90d189e19874b9658aed, ASSIGN because future has completed 2024-11-16T12:51:31,488 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0413c710b53e90d189e19874b9658aed, server=0450ab8807f5,43297,1731761487945}] 2024-11-16T12:51:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T12:51:31,641 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T12:51:31,643 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52621, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T12:51:31,647 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testMasterZKSessionRecoveryFailure_1731761489129,,1731761491290.0413c710b53e90d189e19874b9658aed. 2024-11-16T12:51:31,647 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 0413c710b53e90d189e19874b9658aed, NAME => 'testMasterZKSessionRecoveryFailure_1731761489129,,1731761491290.0413c710b53e90d189e19874b9658aed.', STARTKEY => '', ENDKEY => ''} 2024-11-16T12:51:31,648 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMasterZKSessionRecoveryFailure_1731761489129 0413c710b53e90d189e19874b9658aed 2024-11-16T12:51:31,648 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testMasterZKSessionRecoveryFailure_1731761489129,,1731761491290.0413c710b53e90d189e19874b9658aed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:31,648 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 0413c710b53e90d189e19874b9658aed 2024-11-16T12:51:31,648 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 0413c710b53e90d189e19874b9658aed 2024-11-16T12:51:31,649 INFO [StoreOpener-0413c710b53e90d189e19874b9658aed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family fam of region 0413c710b53e90d189e19874b9658aed 2024-11-16T12:51:31,651 INFO [StoreOpener-0413c710b53e90d189e19874b9658aed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0413c710b53e90d189e19874b9658aed columnFamilyName fam 2024-11-16T12:51:31,651 DEBUG [StoreOpener-0413c710b53e90d189e19874b9658aed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:31,652 INFO [StoreOpener-0413c710b53e90d189e19874b9658aed-1 {}] regionserver.HStore(327): Store=0413c710b53e90d189e19874b9658aed/fam, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:31,652 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 0413c710b53e90d189e19874b9658aed 2024-11-16T12:51:31,653 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testMasterZKSessionRecoveryFailure_1731761489129/0413c710b53e90d189e19874b9658aed 2024-11-16T12:51:31,654 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testMasterZKSessionRecoveryFailure_1731761489129/0413c710b53e90d189e19874b9658aed 2024-11-16T12:51:31,654 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 0413c710b53e90d189e19874b9658aed 2024-11-16T12:51:31,654 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 0413c710b53e90d189e19874b9658aed 2024-11-16T12:51:31,656 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 0413c710b53e90d189e19874b9658aed 2024-11-16T12:51:31,658 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testMasterZKSessionRecoveryFailure_1731761489129/0413c710b53e90d189e19874b9658aed/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:51:31,659 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 0413c710b53e90d189e19874b9658aed; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69903609, jitterRate=0.04164494574069977}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T12:51:31,659 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0413c710b53e90d189e19874b9658aed 2024-11-16T12:51:31,659 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 0413c710b53e90d189e19874b9658aed: Running coprocessor pre-open hook at 1731761491648Writing region info on filesystem at 1731761491648Initializing all the Stores at 1731761491649 (+1 ms)Instantiating store for column family {NAME => 'fam', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761491649Cleaning up temporary data from old regions at 1731761491654 (+5 ms)Running coprocessor post-open hooks at 1731761491659 (+5 ms)Region opened successfully at 1731761491659 2024-11-16T12:51:31,660 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testMasterZKSessionRecoveryFailure_1731761489129,,1731761491290.0413c710b53e90d189e19874b9658aed., pid=6, masterSystemTime=1731761491641 2024-11-16T12:51:31,663 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testMasterZKSessionRecoveryFailure_1731761489129,,1731761491290.0413c710b53e90d189e19874b9658aed. 2024-11-16T12:51:31,663 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testMasterZKSessionRecoveryFailure_1731761489129,,1731761491290.0413c710b53e90d189e19874b9658aed. 2024-11-16T12:51:31,663 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0413c710b53e90d189e19874b9658aed, regionState=OPEN, openSeqNum=2, regionLocation=0450ab8807f5,43297,1731761487945 2024-11-16T12:51:31,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-14-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0413c710b53e90d189e19874b9658aed, server=0450ab8807f5,43297,1731761487945 because future has completed 2024-11-16T12:51:31,671 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-16T12:51:31,671 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 0413c710b53e90d189e19874b9658aed, server=0450ab8807f5,43297,1731761487945 in 180 msec 2024-11-16T12:51:31,674 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-16T12:51:31,674 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testMasterZKSessionRecoveryFailure_1731761489129, region=0413c710b53e90d189e19874b9658aed, ASSIGN in 340 msec 2024-11-16T12:51:31,675 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T12:51:31,675 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testMasterZKSessionRecoveryFailure_1731761489129 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T12:51:31,675 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testMasterZKSessionRecoveryFailure_1731761489129","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731761491675"}]},"ts":"1731761491675"} 2024-11-16T12:51:31,677 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testMasterZKSessionRecoveryFailure_1731761489129, state=ENABLED in hbase:meta 2024-11-16T12:51:31,678 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testMasterZKSessionRecoveryFailure_1731761489129 execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T12:51:31,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testMasterZKSessionRecoveryFailure_1731761489129 in 388 msec 2024-11-16T12:51:31,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T12:51:31,920 INFO [RPCClient-NioEventLoopGroup-6-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testMasterZKSessionRecoveryFailure_1731761489129 completed 2024-11-16T12:51:31,920 INFO [Time-limited test {}] hbase.TestZooKeeper(164): Putting table testMasterZKSessionRecoveryFailure_1731761489129 2024-11-16T12:51:31,923 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testMasterZKSessionRecoveryFailure_1731761489129', row='testrow', locateType=CURRENT is [region=testMasterZKSessionRecoveryFailure_1731761489129,,1731761491290.0413c710b53e90d189e19874b9658aed., hostname=0450ab8807f5,43297,1731761487945, seqNum=2] 2024-11-16T12:51:31,924 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T12:51:31,926 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T12:51:31,932 DEBUG [Time-limited test {}] master.HMaster(3213): Abort called but aborted=true, stopped=true 2024-11-16T12:51:31,932 ERROR [Time-limited test {}] master.HMaster(3218): Master server abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-16T12:51:31,932 ERROR [Time-limited test {}] master.HMaster(3225): ***** ABORTING master 0450ab8807f5,34423,1731761487794: killAll ***** 2024-11-16T12:51:31,932 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master '0450ab8807f5,34423,1731761487794' ***** 2024-11-16T12:51:31,932 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: Stopped by Time-limited test 2024-11-16T12:51:31,932 INFO [M:0;0450ab8807f5:34423 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T12:51:31,932 INFO [M:0;0450ab8807f5:34423 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;0450ab8807f5:34423. 2024-11-16T12:51:31,932 DEBUG [M:0;0450ab8807f5:34423 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:51:31,932 DEBUG [M:0;0450ab8807f5:34423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:31,932 INFO [M:0;0450ab8807f5:34423 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T12:51:31,932 DEBUG [M:0;0450ab8807f5:34423 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T12:51:31,933 DEBUG [M:0;0450ab8807f5:34423 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T12:51:31,933 DEBUG [master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.large.0-1731761490866 {}] cleaner.HFileCleaner(306): Exit Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.large.0-1731761490866,5,FailOnTimeoutGroup] 2024-11-16T12:51:31,933 INFO [M:0;0450ab8807f5:34423 {}] hbase.ChoreService(370): Chore service for: master/0450ab8807f5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T12:51:31,933 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T12:51:31,933 INFO [M:0;0450ab8807f5:34423 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T12:51:31,933 DEBUG [M:0;0450ab8807f5:34423 {}] master.HMaster(1795): Stopping service threads 2024-11-16T12:51:31,933 INFO [M:0;0450ab8807f5:34423 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T12:51:31,933 DEBUG [master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.small.0-1731761490866 {}] cleaner.HFileCleaner(306): Exit Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.small.0-1731761490866,5,FailOnTimeoutGroup] 2024-11-16T12:51:31,933 INFO [M:0;0450ab8807f5:34423 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T12:51:31,933 INFO [M:0;0450ab8807f5:34423 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=true 2024-11-16T12:51:31,933 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T12:51:31,958 DEBUG [HMaster-EventLoopGroup-11-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0450ab8807f5,34423,1731761487794 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:31,958 DEBUG [HMaster-EventLoopGroup-11-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0450ab8807f5,34423,1731761487794 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:31,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45447-0x10144fc6ca6000f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:31,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43297-0x10144fc6ca6000e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:31,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:31,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43297-0x10144fc6ca6000e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:31,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:31,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45447-0x10144fc6ca6000f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:31,970 DEBUG [RS:1;0450ab8807f5:45447 {}] regionserver.HRegionServer(2589): No master found; retry 2024-11-16T12:51:31,971 DEBUG [M:0;0450ab8807f5:34423 {}] zookeeper.ZKUtil(347): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-16T12:51:31,971 DEBUG [RS:0;0450ab8807f5:43297 {}] regionserver.HRegionServer(2589): No master found; retry 2024-11-16T12:51:31,971 WARN [M:0;0450ab8807f5:34423 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-16T12:51:31,971 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 0450ab8807f5,43297,1731761487945: killAll ***** 2024-11-16T12:51:31,971 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-16T12:51:31,971 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-16T12:51:31,971 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45447-0x10144fc6ca6000f, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:31,971 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43297-0x10144fc6ca6000e, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:31,971 INFO [M:0;0450ab8807f5:34423 {}] master.ServerManager(1134): Rewriting .lastflushedseqids file at: hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/.lastflushedseqids 2024-11-16T12:51:31,971 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-16T12:51:31,972 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-16T12:51:31,973 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-16T12:51:31,974 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 261964720 }, "NonHeapMemoryUsage": { "committed": 162136064, "init": 7667712, "max": -1, "used": 158182592 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "0450ab8807f5", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 13406, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 47, "ProcessCallTime_min": 0, "ProcessCallTime_max": 5, "ProcessCallTime_mean": 1, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 1, "ProcessCallTime_75th_percentile": 1, "ProcessCallTime_90th_percentile": 3, "ProcessCallTime_95th_percentile": 4, "ProcessCallTime_98th_percentile": 5, "ProcessCallTime_99th_percentile": 5, "ProcessCallTime_99.9th_percentile": 5, "ProcessCallTime_TimeRangeCount_0-1": 17, "ProcessCallTime_TimeRangeCount_1-3": 2, "ProcessCallTime_TimeRangeCount_3-10": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 47, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 1, "QueueCallTime_95th_percentile": 1, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 20, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 47, "TotalCallTime_min": 1, "TotalCallTime_max": 5, "TotalCallTime_mean": 1, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 1, "TotalCallTime_75th_percentile": 1, "TotalCallTime_90th_percentile": 3, "TotalCallTime_95th_percentile": 4, "TotalCallTime_98th_percentile": 5, "TotalCallTime_99th_percentile": 5, "TotalCallTime_99.9th_percentile": 5, "TotalCallTime_TimeRangeCount_0-1": 16, "TotalCallTime_TimeRangeCount_1-3": 3, "TotalCallTime_TimeRangeCount_3-10": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 47, "ResponseSize_min": 0, "ResponseSize_max": 1098, "ResponseSize_mean": 86, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 3, "ResponseSize_75th_percentile": 23, "ResponseSize_90th_percentile": 175, "ResponseSize_95th_percentile": 176, "ResponseSize_98th_percentile": 760, "ResponseSize_99th_percentile": 929, "ResponseSize_99.9th_percentile": 1081, "ResponseSize_SizeRangeCount_0-10": 12, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 4, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 47, "RequestSize_min": 89, "RequestSize_max": 382, "RequestSize_mean": 151, "RequestSize_25th_percentile": 100, "RequestSize_median": 121, "RequestSize_75th_percentile": 131, "RequestSize_90th_percentile": 274, "RequestSize_95th_percentile": 323, "RequestSize_98th_percentile": 366, "RequestSize_99th_percentile": 374, "RequestSize_99.9th_percentile": 381, "RequestSize_SizeRangeCount_10-100": 5, "RequestSize_SizeRangeCount_100-1000": 15, "sentBytes": 3296 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "0450ab8807f5", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:53208", "tag.serverName": "0450ab8807f5,36743,1731761465323", "tag.clusterId": "be09e536-22d9-488d-a733-7968a5bb48e1", "tag.Context": "regionserver", "tag.Hostname": "0450ab8807f5", "regionCount": 0, "storeCount": 0, "hlogFileCount": 7, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731761465323, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 55730, "localBytesRead": 55730, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 32470, "ByteBuffAllocatorPoolAllocationBytes": 66560, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 7, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 1, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 6, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 1, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 6, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 5153, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 74, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 1, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-16T12:51:31,975 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0450ab8807f5,43297,1731761487945' ***** 2024-11-16T12:51:31,975 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: killAll 2024-11-16T12:51:31,975 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 0450ab8807f5,45447,1731761487981: killAll ***** 2024-11-16T12:51:31,975 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-16T12:51:31,975 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-16T12:51:31,976 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-16T12:51:31,976 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-16T12:51:31,976 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-16T12:51:31,978 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 261964720 }, "NonHeapMemoryUsage": { "committed": 162136064, "init": 7667712, "max": -1, "used": 158202928 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "0450ab8807f5", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 13406, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 47, "ProcessCallTime_min": 0, "ProcessCallTime_max": 5, "ProcessCallTime_mean": 1, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 1, "ProcessCallTime_75th_percentile": 1, "ProcessCallTime_90th_percentile": 3, "ProcessCallTime_95th_percentile": 4, "ProcessCallTime_98th_percentile": 5, "ProcessCallTime_99th_percentile": 5, "ProcessCallTime_99.9th_percentile": 5, "ProcessCallTime_TimeRangeCount_0-1": 17, "ProcessCallTime_TimeRangeCount_1-3": 2, "ProcessCallTime_TimeRangeCount_3-10": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 47, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 1, "QueueCallTime_95th_percentile": 1, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 20, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 47, "TotalCallTime_min": 1, "TotalCallTime_max": 5, "TotalCallTime_mean": 1, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 1, "TotalCallTime_75th_percentile": 1, "TotalCallTime_90th_percentile": 3, "TotalCallTime_95th_percentile": 4, "TotalCallTime_98th_percentile": 5, "TotalCallTime_99th_percentile": 5, "TotalCallTime_99.9th_percentile": 5, "TotalCallTime_TimeRangeCount_0-1": 16, "TotalCallTime_TimeRangeCount_1-3": 3, "TotalCallTime_TimeRangeCount_3-10": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 47, "ResponseSize_min": 0, "ResponseSize_max": 1098, "ResponseSize_mean": 86, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 3, "ResponseSize_75th_percentile": 23, "ResponseSize_90th_percentile": 175, "ResponseSize_95th_percentile": 176, "ResponseSize_98th_percentile": 760, "ResponseSize_99th_percentile": 929, "ResponseSize_99.9th_percentile": 1081, "ResponseSize_SizeRangeCount_0-10": 12, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 4, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 47, "RequestSize_min": 89, "RequestSize_max": 382, "RequestSize_mean": 151, "RequestSize_25th_percentile": 100, "RequestSize_median": 121, "RequestSize_75th_percentile": 131, "RequestSize_90th_percentile": 274, "RequestSize_95th_percentile": 323, "RequestSize_98th_percentile": 366, "RequestSize_99th_percentile": 374, "RequestSize_99.9th_percentile": 381, "RequestSize_SizeRangeCount_10-100": 5, "RequestSize_SizeRangeCount_100-1000": 15, "sentBytes": 3296 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "0450ab8807f5", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:53208", "tag.serverName": "0450ab8807f5,36743,1731761465323", "tag.clusterId": "be09e536-22d9-488d-a733-7968a5bb48e1", "tag.Context": "regionserver", "tag.Hostname": "0450ab8807f5", "regionCount": 0, "storeCount": 0, "hlogFileCount": 7, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731761465323, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 55730, "localBytesRead": 55730, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 32470, "ByteBuffAllocatorPoolAllocationBytes": 66560, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 7, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 1, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 6, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 1, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 6, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 5153, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 74, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 1, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-16T12:51:31,978 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0450ab8807f5,45447,1731761487981' ***** 2024-11-16T12:51:31,978 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: killAll 2024-11-16T12:51:31,979 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T12:51:31,979 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.TestZooKeeper.after(TestZooKeeper.java:110) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:51:31,979 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:31,979 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2d714c78 to 127.0.0.1:53208 2024-11-16T12:51:31,979 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-16T12:51:31,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741881_1059 (size=147) 2024-11-16T12:51:31,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741881_1059 (size=147) 2024-11-16T12:51:31,981 INFO [M:0;0450ab8807f5:34423 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T12:51:31,982 INFO [M:0;0450ab8807f5:34423 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=true 2024-11-16T12:51:31,984 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T12:51:31,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741875_1053 (size=17906) 2024-11-16T12:51:31,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741875_1053 (size=17906) 2024-11-16T12:51:31,986 DEBUG [M:0;0450ab8807f5:34423 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T12:51:31,986 INFO [M:0;0450ab8807f5:34423 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:31,986 DEBUG [M:0;0450ab8807f5:34423 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:31,986 DEBUG [M:0;0450ab8807f5:34423 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T12:51:31,987 DEBUG [M:0;0450ab8807f5:34423 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:31,988 ERROR [M:0;0450ab8807f5:34423 {}] regionserver.HRegion(1960): Memstore data size is 16069 in region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:31,988 INFO [M:0;0450ab8807f5:34423 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:31,988 DEBUG [M:0;0450ab8807f5:34423 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731761491986Disabling compacts and flushes for region at 1731761491986Disabling writes for close at 1731761491986Writing region close event to WAL at 1731761491988 (+2 ms)Closed at 1731761491988 2024-11-16T12:51:31,988 INFO [M:0;0450ab8807f5:34423 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-16T12:51:31,988 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T12:51:31,988 INFO [M:0;0450ab8807f5:34423 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34423 2024-11-16T12:51:31,989 INFO [M:0;0450ab8807f5:34423 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T12:51:31,989 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T12:51:32,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:32,095 INFO [M:0;0450ab8807f5:34423 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T12:51:32,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34423-0x10144fc6ca6000c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:32,171 INFO [RS:1;0450ab8807f5:45447 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T12:51:32,171 INFO [RS:0;0450ab8807f5:43297 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T12:51:32,171 INFO [RS:1;0450ab8807f5:45447 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-16T12:51:32,171 INFO [RS:1;0450ab8807f5:45447 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-16T12:51:32,171 INFO [RS:0;0450ab8807f5:43297 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-16T12:51:32,171 INFO [RS:1;0450ab8807f5:45447 {}] regionserver.HRegionServer(956): aborting server 0450ab8807f5,45447,1731761487981 2024-11-16T12:51:32,171 INFO [RS:0;0450ab8807f5:43297 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-16T12:51:32,171 INFO [RS:1;0450ab8807f5:45447 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T12:51:32,171 INFO [RS:0;0450ab8807f5:43297 {}] regionserver.HRegionServer(3091): Received CLOSE for 0413c710b53e90d189e19874b9658aed 2024-11-16T12:51:32,171 INFO [RS:1;0450ab8807f5:45447 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;0450ab8807f5:45447. 2024-11-16T12:51:32,171 DEBUG [RS:1;0450ab8807f5:45447 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:51:32,171 DEBUG [RS:1;0450ab8807f5:45447 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:32,171 INFO [RS:0;0450ab8807f5:43297 {}] regionserver.HRegionServer(956): aborting server 0450ab8807f5,43297,1731761487945 2024-11-16T12:51:32,171 INFO [RS:1;0450ab8807f5:45447 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T12:51:32,171 INFO [RS:0;0450ab8807f5:43297 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T12:51:32,172 INFO [RS:1;0450ab8807f5:45447 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T12:51:32,172 INFO [RS:1;0450ab8807f5:45447 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T12:51:32,172 INFO [RS:0;0450ab8807f5:43297 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0450ab8807f5:43297. 2024-11-16T12:51:32,172 INFO [RS:1;0450ab8807f5:45447 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T12:51:32,172 DEBUG [RS:0;0450ab8807f5:43297 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:51:32,172 DEBUG [RS:0;0450ab8807f5:43297 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:32,172 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0413c710b53e90d189e19874b9658aed, disabling compactions & flushes 2024-11-16T12:51:32,172 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testMasterZKSessionRecoveryFailure_1731761489129,,1731761491290.0413c710b53e90d189e19874b9658aed. 2024-11-16T12:51:32,172 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testMasterZKSessionRecoveryFailure_1731761489129,,1731761491290.0413c710b53e90d189e19874b9658aed. 2024-11-16T12:51:32,172 INFO [RS:0;0450ab8807f5:43297 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-16T12:51:32,172 DEBUG [RS:0;0450ab8807f5:43297 {}] regionserver.HRegionServer(1325): Online Regions={0413c710b53e90d189e19874b9658aed=testMasterZKSessionRecoveryFailure_1731761489129,,1731761491290.0413c710b53e90d189e19874b9658aed.} 2024-11-16T12:51:32,172 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testMasterZKSessionRecoveryFailure_1731761489129,,1731761491290.0413c710b53e90d189e19874b9658aed. after waiting 0 ms 2024-11-16T12:51:32,172 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testMasterZKSessionRecoveryFailure_1731761489129,,1731761491290.0413c710b53e90d189e19874b9658aed. 2024-11-16T12:51:32,172 DEBUG [RS:0;0450ab8807f5:43297 {}] regionserver.HRegionServer(1351): Waiting on 0413c710b53e90d189e19874b9658aed 2024-11-16T12:51:32,174 INFO [RS:1;0450ab8807f5:45447 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-16T12:51:32,174 DEBUG [RS:1;0450ab8807f5:45447 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-16T12:51:32,175 DEBUG [RS:1;0450ab8807f5:45447 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T12:51:32,175 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T12:51:32,175 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T12:51:32,175 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T12:51:32,175 ERROR [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1960): Memstore data size is 41 in region testMasterZKSessionRecoveryFailure_1731761489129,,1731761491290.0413c710b53e90d189e19874b9658aed. 2024-11-16T12:51:32,175 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T12:51:32,175 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T12:51:32,175 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testMasterZKSessionRecoveryFailure_1731761489129,,1731761491290.0413c710b53e90d189e19874b9658aed. 2024-11-16T12:51:32,175 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0413c710b53e90d189e19874b9658aed: Waiting for close lock at 1731761492172Running coprocessor pre-close hooks at 1731761492172Disabling compacts and flushes for region at 1731761492172Disabling writes for close at 1731761492172Writing region close event to WAL at 1731761492175 (+3 ms)Running coprocessor post-close hooks at 1731761492175Closed at 1731761492175 2024-11-16T12:51:32,175 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testMasterZKSessionRecoveryFailure_1731761489129,,1731761491290.0413c710b53e90d189e19874b9658aed. 2024-11-16T12:51:32,175 ERROR [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 1857 in region hbase:meta,,1.1588230740 2024-11-16T12:51:32,176 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T12:51:32,176 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T12:51:32,176 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731761492174Running coprocessor pre-close hooks at 1731761492175 (+1 ms)Disabling compacts and flushes for region at 1731761492175Disabling writes for close at 1731761492175Writing region close event to WAL at 1731761492175Running coprocessor post-close hooks at 1731761492175Closed at 1731761492176 (+1 ms) 2024-11-16T12:51:32,176 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T12:51:32,185 INFO [regionserver/0450ab8807f5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:51:32,189 INFO [regionserver/0450ab8807f5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:51:32,372 INFO [RS:0;0450ab8807f5:43297 {}] regionserver.HRegionServer(976): stopping server 0450ab8807f5,43297,1731761487945; all regions closed. 2024-11-16T12:51:32,375 INFO [RS:1;0450ab8807f5:45447 {}] regionserver.HRegionServer(976): stopping server 0450ab8807f5,45447,1731761487981; all regions closed. 2024-11-16T12:51:32,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741871_1049 (size=623) 2024-11-16T12:51:32,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741871_1049 (size=623) 2024-11-16T12:51:32,376 DEBUG [RS:0;0450ab8807f5:43297 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:32,376 INFO [RS:0;0450ab8807f5:43297 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:51:32,376 INFO [RS:0;0450ab8807f5:43297 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T12:51:32,376 INFO [RS:0;0450ab8807f5:43297 {}] hbase.ChoreService(370): Chore service for: regionserver/0450ab8807f5:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-16T12:51:32,376 INFO [RS:0;0450ab8807f5:43297 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T12:51:32,376 INFO [regionserver/0450ab8807f5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T12:51:32,376 INFO [RS:0;0450ab8807f5:43297 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T12:51:32,376 INFO [RS:0;0450ab8807f5:43297 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T12:51:32,376 INFO [RS:0;0450ab8807f5:43297 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T12:51:32,376 INFO [RS:0;0450ab8807f5:43297 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43297 2024-11-16T12:51:32,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741872_1050 (size=2499) 2024-11-16T12:51:32,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741872_1050 (size=2499) 2024-11-16T12:51:32,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741870_1048 (size=93) 2024-11-16T12:51:32,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741870_1048 (size=93) 2024-11-16T12:51:32,381 DEBUG [RS:1;0450ab8807f5:45447 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:32,381 INFO [RS:1;0450ab8807f5:45447 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:51:32,381 INFO [RS:1;0450ab8807f5:45447 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T12:51:32,382 INFO [RS:1;0450ab8807f5:45447 {}] hbase.ChoreService(370): Chore service for: regionserver/0450ab8807f5:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-16T12:51:32,382 INFO [RS:1;0450ab8807f5:45447 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T12:51:32,382 INFO [regionserver/0450ab8807f5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T12:51:32,382 INFO [RS:1;0450ab8807f5:45447 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45447 2024-11-16T12:51:32,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43297-0x10144fc6ca6000e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0450ab8807f5,43297,1731761487945 2024-11-16T12:51:32,404 INFO [RS:0;0450ab8807f5:43297 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T12:51:32,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45447-0x10144fc6ca6000f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0450ab8807f5,45447,1731761487981 2024-11-16T12:51:32,412 INFO [RS:1;0450ab8807f5:45447 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T12:51:32,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43297-0x10144fc6ca6000e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:32,520 INFO [RS:0;0450ab8807f5:43297 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T12:51:32,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43297-0x10144fc6ca6000e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:32,520 INFO [RS:0;0450ab8807f5:43297 {}] regionserver.HRegionServer(1031): Exiting; stopping=0450ab8807f5,43297,1731761487945; zookeeper connection closed. 2024-11-16T12:51:32,521 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3100812d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3100812d 2024-11-16T12:51:32,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45447-0x10144fc6ca6000f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:32,529 INFO [RS:1;0450ab8807f5:45447 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T12:51:32,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45447-0x10144fc6ca6000f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:32,529 INFO [RS:1;0450ab8807f5:45447 {}] regionserver.HRegionServer(1031): Exiting; stopping=0450ab8807f5,45447,1731761487981; zookeeper connection closed. 2024-11-16T12:51:32,529 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6f787d47 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6f787d47 2024-11-16T12:51:32,529 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 2 master(s) and 2 regionserver(s) complete 2024-11-16T12:51:32,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6000b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:32,661 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:32,665 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=testing utility connecting to ZooKeeper ensemble=127.0.0.1:53208 2024-11-16T12:51:32,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility0x0, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:51:32,696 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): testing utility-0x10144fc6ca60011 connected 2024-11-16T12:51:32,696 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-16T12:51:32,697 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/meta-region-server 2024-11-16T12:51:32,697 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/rs 2024-11-16T12:51:32,697 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/splitWAL 2024-11-16T12:51:32,698 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters 2024-11-16T12:51:32,698 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc 2024-11-16T12:51:32,698 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/draining 2024-11-16T12:51:32,698 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/master-maintenance 2024-11-16T12:51:32,699 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot 2024-11-16T12:51:32,699 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/hbaseid 2024-11-16T12:51:32,700 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/table 2024-11-16T12:51:32,702 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc/abort 2024-11-16T12:51:32,703 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc/acquired 2024-11-16T12:51:32,703 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc/reached 2024-11-16T12:51:32,704 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot/abort 2024-11-16T12:51:32,705 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot/acquired 2024-11-16T12:51:32,705 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot/reached 2024-11-16T12:51:32,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/reached 2024-11-16T12:51:32,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/online-snapshot 2024-11-16T12:51:32,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/acquired 2024-11-16T12:51:32,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/abort 2024-11-16T12:51:32,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/reached 2024-11-16T12:51:32,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/flush-table-proc 2024-11-16T12:51:32,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/acquired 2024-11-16T12:51:32,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/abort 2024-11-16T12:51:32,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/table 2024-11-16T12:51:32,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:32,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/hbaseid 2024-11-16T12:51:32,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot 2024-11-16T12:51:32,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master-maintenance 2024-11-16T12:51:32,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/draining 2024-11-16T12:51:32,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc 2024-11-16T12:51:32,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:32,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/splitWAL 2024-11-16T12:51:32,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs 2024-11-16T12:51:32,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:32,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T12:51:32,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase 2024-11-16T12:51:32,723 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestZooKeeper#testMasterZKSessionRecoveryFailure Thread=358 (was 332) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-14-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-11-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-11-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-14-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1205652929) connection to localhost/127.0.0.1:44975 from jenkins.hfs.5 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:44975 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-11-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-12-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_688061434_22 at /127.0.0.1:50302 [Waiting for operation #31] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1694157692_22 at /127.0.0.1:36028 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-12-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1205652929) connection to localhost/127.0.0.1:44975 from jenkins.hfs.4 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: HMaster-EventLoopGroup-12-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-14-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:44975 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=630 (was 594) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=237 (was 232) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3284 (was 3261) - AvailableMemoryMB LEAK? - 2024-11-16T12:51:32,734 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestZooKeeper#testLogSplittingAfterMasterRecoveryDueToZKExpiry Thread=358, OpenFileDescriptor=630, MaxFileDescriptor=1048576, SystemLoadAverage=237, ProcessCount=11, AvailableMemoryMB=3284 2024-11-16T12:51:32,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741882_1060 (size=7) 2024-11-16T12:51:32,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741882_1060 (size=7) 2024-11-16T12:51:32,742 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b with version=8 2024-11-16T12:51:32,742 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/hbase-staging 2024-11-16T12:51:32,744 INFO [Time-limited test {}] client.ConnectionUtils(128): master/0450ab8807f5:0 server-side Connection retries=45 2024-11-16T12:51:32,744 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:32,744 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:32,744 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T12:51:32,744 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:32,744 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T12:51:32,744 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T12:51:32,745 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T12:51:32,745 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40581 2024-11-16T12:51:32,746 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40581 connecting to ZooKeeper ensemble=127.0.0.1:53208 2024-11-16T12:51:32,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:405810x0, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:51:32,754 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40581-0x10144fc6ca60012 connected 2024-11-16T12:51:32,812 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:32,813 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:32,815 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40581-0x10144fc6ca60012, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:51:32,815 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b, hbase.cluster.distributed=false 2024-11-16T12:51:32,816 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40581-0x10144fc6ca60012, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T12:51:32,817 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40581 2024-11-16T12:51:32,817 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40581 2024-11-16T12:51:32,817 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40581 2024-11-16T12:51:32,818 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40581 2024-11-16T12:51:32,818 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40581 2024-11-16T12:51:32,819 INFO [Time-limited test {}] client.ConnectionUtils(128): master/0450ab8807f5:0 server-side Connection retries=45 2024-11-16T12:51:32,819 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:32,819 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:32,819 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T12:51:32,819 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:32,819 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T12:51:32,819 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T12:51:32,819 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T12:51:32,819 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44015 2024-11-16T12:51:32,820 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44015 connecting to ZooKeeper ensemble=127.0.0.1:53208 2024-11-16T12:51:32,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:440150x0, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:51:32,829 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44015-0x10144fc6ca60013 connected 2024-11-16T12:51:32,839 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:32,840 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:32,842 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:51:32,842 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b, hbase.cluster.distributed=false 2024-11-16T12:51:32,843 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T12:51:32,843 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44015 2024-11-16T12:51:32,843 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44015 2024-11-16T12:51:32,844 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44015 2024-11-16T12:51:32,844 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44015 2024-11-16T12:51:32,844 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44015 2024-11-16T12:51:32,857 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0450ab8807f5:0 server-side Connection retries=45 2024-11-16T12:51:32,857 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:32,857 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:32,857 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T12:51:32,857 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:32,857 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T12:51:32,857 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T12:51:32,858 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T12:51:32,859 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32881 2024-11-16T12:51:32,860 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:32881 connecting to ZooKeeper ensemble=127.0.0.1:53208 2024-11-16T12:51:32,860 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:32,861 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:32,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:328810x0, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:51:32,871 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:328810x0, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:51:32,871 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:32881-0x10144fc6ca60014 connected 2024-11-16T12:51:32,871 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T12:51:32,872 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T12:51:32,872 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32881-0x10144fc6ca60014, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:32,873 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32881-0x10144fc6ca60014, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T12:51:32,874 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32881 2024-11-16T12:51:32,874 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32881 2024-11-16T12:51:32,874 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32881 2024-11-16T12:51:32,875 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32881 2024-11-16T12:51:32,875 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32881 2024-11-16T12:51:32,894 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0450ab8807f5:0 server-side Connection retries=45 2024-11-16T12:51:32,894 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:32,895 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:32,895 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T12:51:32,895 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:32,895 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T12:51:32,895 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T12:51:32,895 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T12:51:32,896 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33961 2024-11-16T12:51:32,897 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33961 connecting to ZooKeeper ensemble=127.0.0.1:53208 2024-11-16T12:51:32,898 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:32,900 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:32,912 DEBUG [pool-593-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:339610x0, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:51:32,912 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:339610x0, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:51:32,912 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33961-0x10144fc6ca60015 connected 2024-11-16T12:51:32,913 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T12:51:32,913 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T12:51:32,914 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33961-0x10144fc6ca60015, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:32,915 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33961-0x10144fc6ca60015, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T12:51:32,917 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33961 2024-11-16T12:51:32,918 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33961 2024-11-16T12:51:32,918 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33961 2024-11-16T12:51:32,919 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33961 2024-11-16T12:51:32,919 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33961 2024-11-16T12:51:32,934 DEBUG [M:0;0450ab8807f5:40581 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0450ab8807f5:40581 2024-11-16T12:51:32,934 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0450ab8807f5,40581,1731761492744 2024-11-16T12:51:32,937 DEBUG [M:1;0450ab8807f5:44015 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:1;0450ab8807f5:44015 2024-11-16T12:51:32,937 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0450ab8807f5,44015,1731761492818 2024-11-16T12:51:32,945 DEBUG [pool-593-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10144fc6ca60015, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:32,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40581-0x10144fc6ca60012, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:32,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:32,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32881-0x10144fc6ca60014, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:32,953 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40581-0x10144fc6ca60012, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0450ab8807f5,40581,1731761492744 2024-11-16T12:51:32,954 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0450ab8807f5,44015,1731761492818 2024-11-16T12:51:32,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40581-0x10144fc6ca60012, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:32,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32881-0x10144fc6ca60014, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:32,962 DEBUG [pool-593-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10144fc6ca60015, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:32,962 DEBUG [pool-593-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10144fc6ca60015, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:32,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32881-0x10144fc6ca60014, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:32,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:32,970 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40581-0x10144fc6ca60012, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T12:51:32,970 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T12:51:32,970 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0450ab8807f5,40581,1731761492744 from backup master directory 2024-11-16T12:51:32,971 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ActiveMasterManager(296): Another master is the active master, 0450ab8807f5,40581,1731761492744; waiting to become the next active master 2024-11-16T12:51:32,978 DEBUG [pool-593-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10144fc6ca60015, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:32,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40581-0x10144fc6ca60012, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0450ab8807f5,40581,1731761492744 2024-11-16T12:51:32,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:32,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32881-0x10144fc6ca60014, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:32,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40581-0x10144fc6ca60012, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:32,979 WARN [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T12:51:32,979 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0450ab8807f5,40581,1731761492744 2024-11-16T12:51:32,983 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/hbase.id] with ID: 275eeb35-c30e-47fc-a490-ffb4d908b232 2024-11-16T12:51:32,983 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/.tmp/hbase.id 2024-11-16T12:51:32,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741883_1061 (size=42) 2024-11-16T12:51:32,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741883_1061 (size=42) 2024-11-16T12:51:32,989 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/.tmp/hbase.id]:[hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/hbase.id] 2024-11-16T12:51:33,001 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:33,001 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T12:51:33,002 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-16T12:51:33,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:33,012 DEBUG [pool-593-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10144fc6ca60015, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:33,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32881-0x10144fc6ca60014, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:33,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40581-0x10144fc6ca60012, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:33,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741884_1062 (size=196) 2024-11-16T12:51:33,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741884_1062 (size=196) 2024-11-16T12:51:33,019 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T12:51:33,020 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T12:51:33,020 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:51:33,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741885_1063 (size=1189) 2024-11-16T12:51:33,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741885_1063 (size=1189) 2024-11-16T12:51:33,029 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store 2024-11-16T12:51:33,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741886_1064 (size=34) 2024-11-16T12:51:33,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741886_1064 (size=34) 2024-11-16T12:51:33,035 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:33,036 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T12:51:33,036 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:33,036 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:33,036 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T12:51:33,036 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:33,036 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:33,036 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731761493036Disabling compacts and flushes for region at 1731761493036Disabling writes for close at 1731761493036Writing region close event to WAL at 1731761493036Closed at 1731761493036 2024-11-16T12:51:33,036 WARN [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/.initializing 2024-11-16T12:51:33,037 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,40581,1731761492744 2024-11-16T12:51:33,038 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T12:51:33,040 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0450ab8807f5%2C40581%2C1731761492744, suffix=, logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,40581,1731761492744, archiveDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/oldWALs, maxLogs=10 2024-11-16T12:51:33,052 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,40581,1731761492744/0450ab8807f5%2C40581%2C1731761492744.1731761493040, exclude list is [], retry=0 2024-11-16T12:51:33,055 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43257,DS-06d91055-02a4-40d7-bddb-15aee81f02ac,DISK] 2024-11-16T12:51:33,055 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36961,DS-7ef9e9a6-250e-467c-989b-a9ffcab2299e,DISK] 2024-11-16T12:51:33,057 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,40581,1731761492744/0450ab8807f5%2C40581%2C1731761492744.1731761493040 2024-11-16T12:51:33,057 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:46667:46667)] 2024-11-16T12:51:33,057 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-16T12:51:33,057 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:33,057 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:33,057 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:33,059 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:33,060 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-16T12:51:33,060 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:33,061 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:33,061 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:33,062 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-16T12:51:33,062 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:33,063 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:33,063 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:33,064 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-16T12:51:33,064 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:33,065 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:33,065 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:33,066 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-16T12:51:33,066 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:33,067 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:33,067 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:33,068 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:33,068 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:33,069 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:33,069 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:33,070 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T12:51:33,071 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:33,074 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:51:33,074 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65095146, jitterRate=-0.030006736516952515}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T12:51:33,075 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731761493057Initializing all the Stores at 1731761493058 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761493058Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761493059 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761493059Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761493059Cleaning up temporary data from old regions at 1731761493069 (+10 ms)Region opened successfully at 1731761493075 (+6 ms) 2024-11-16T12:51:33,075 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-16T12:51:33,079 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@261a2c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0450ab8807f5/172.17.0.2:0 2024-11-16T12:51:33,080 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-16T12:51:33,080 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-16T12:51:33,080 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-16T12:51:33,080 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T12:51:33,081 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T12:51:33,081 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-16T12:51:33,081 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-16T12:51:33,084 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-16T12:51:33,084 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40581-0x10144fc6ca60012, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-16T12:51:33,111 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-16T12:51:33,111 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-16T12:51:33,112 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40581-0x10144fc6ca60012, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-16T12:51:33,120 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-16T12:51:33,120 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-16T12:51:33,121 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40581-0x10144fc6ca60012, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-16T12:51:33,128 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-16T12:51:33,129 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40581-0x10144fc6ca60012, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-16T12:51:33,137 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-16T12:51:33,139 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40581-0x10144fc6ca60012, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-16T12:51:33,145 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-16T12:51:33,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T12:51:33,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40581-0x10144fc6ca60012, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T12:51:33,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32881-0x10144fc6ca60014, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T12:51:33,153 DEBUG [pool-593-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10144fc6ca60015, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T12:51:33,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:33,153 DEBUG [pool-593-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10144fc6ca60015, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:33,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32881-0x10144fc6ca60014, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:33,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40581-0x10144fc6ca60012, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:33,154 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0450ab8807f5,40581,1731761492744, sessionid=0x10144fc6ca60012, setting cluster-up flag (Was=false) 2024-11-16T12:51:33,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:33,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40581-0x10144fc6ca60012, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:33,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32881-0x10144fc6ca60014, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:33,189 DEBUG [pool-593-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10144fc6ca60015, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:33,220 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-16T12:51:33,221 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0450ab8807f5,40581,1731761492744 2024-11-16T12:51:33,237 DEBUG [pool-593-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10144fc6ca60015, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:33,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:33,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40581-0x10144fc6ca60012, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:33,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32881-0x10144fc6ca60014, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:33,262 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-16T12:51:33,263 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0450ab8807f5,40581,1731761492744 2024-11-16T12:51:33,264 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-16T12:51:33,266 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T12:51:33,266 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-16T12:51:33,267 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0450ab8807f5,40581,1731761492744 Number of backup masters: 1 0450ab8807f5,44015,1731761492818 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-16T12:51:33,268 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:33,268 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:33,268 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:33,268 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:33,268 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0450ab8807f5:0, corePoolSize=10, maxPoolSize=10 2024-11-16T12:51:33,268 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:33,268 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0450ab8807f5:0, corePoolSize=2, maxPoolSize=2 2024-11-16T12:51:33,268 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:33,275 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731761523275 2024-11-16T12:51:33,275 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-16T12:51:33,275 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-16T12:51:33,275 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-16T12:51:33,275 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-16T12:51:33,275 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-16T12:51:33,275 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-16T12:51:33,275 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T12:51:33,275 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-16T12:51:33,275 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:33,276 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:33,276 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-16T12:51:33,279 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-16T12:51:33,279 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-16T12:51:33,279 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-16T12:51:33,282 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-16T12:51:33,282 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-16T12:51:33,282 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.large.0-1731761493282,5,FailOnTimeoutGroup] 2024-11-16T12:51:33,282 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.small.0-1731761493282,5,FailOnTimeoutGroup] 2024-11-16T12:51:33,282 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:33,282 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-16T12:51:33,282 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:33,282 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:33,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741888_1066 (size=1321) 2024-11-16T12:51:33,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741888_1066 (size=1321) 2024-11-16T12:51:33,288 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-16T12:51:33,288 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:33,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741889_1067 (size=32) 2024-11-16T12:51:33,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741889_1067 (size=32) 2024-11-16T12:51:33,295 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:33,296 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T12:51:33,297 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T12:51:33,297 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:33,298 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:33,298 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T12:51:33,299 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T12:51:33,299 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:33,300 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:33,300 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T12:51:33,301 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T12:51:33,301 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:33,301 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:33,302 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T12:51:33,303 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T12:51:33,303 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:33,304 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:33,304 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T12:51:33,304 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740 2024-11-16T12:51:33,305 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740 2024-11-16T12:51:33,306 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T12:51:33,306 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T12:51:33,306 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T12:51:33,307 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T12:51:33,309 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:51:33,310 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62285312, jitterRate=-0.07187652587890625}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T12:51:33,310 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731761493295Initializing all the Stores at 1731761493296 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761493296Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761493296Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761493296Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761493296Cleaning up temporary data from old regions at 1731761493306 (+10 ms)Region opened successfully at 1731761493310 (+4 ms) 2024-11-16T12:51:33,310 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T12:51:33,310 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T12:51:33,310 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T12:51:33,310 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T12:51:33,310 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T12:51:33,311 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T12:51:33,311 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731761493310Disabling compacts and flushes for region at 1731761493310Disabling writes for close at 1731761493310Writing region close event to WAL at 1731761493311 (+1 ms)Closed at 1731761493311 2024-11-16T12:51:33,312 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T12:51:33,312 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-16T12:51:33,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T12:51:33,313 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T12:51:33,314 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-16T12:51:33,324 INFO [RS:1;0450ab8807f5:33961 {}] regionserver.HRegionServer(746): ClusterId : 275eeb35-c30e-47fc-a490-ffb4d908b232 2024-11-16T12:51:33,324 INFO [RS:0;0450ab8807f5:32881 {}] regionserver.HRegionServer(746): ClusterId : 275eeb35-c30e-47fc-a490-ffb4d908b232 2024-11-16T12:51:33,324 DEBUG [RS:1;0450ab8807f5:33961 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T12:51:33,324 DEBUG [RS:0;0450ab8807f5:32881 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T12:51:33,345 DEBUG [RS:0;0450ab8807f5:32881 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T12:51:33,345 DEBUG [RS:1;0450ab8807f5:33961 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T12:51:33,345 DEBUG [RS:0;0450ab8807f5:32881 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T12:51:33,345 DEBUG [RS:1;0450ab8807f5:33961 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T12:51:33,354 DEBUG [RS:1;0450ab8807f5:33961 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T12:51:33,354 DEBUG [RS:0;0450ab8807f5:32881 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T12:51:33,355 DEBUG [RS:1;0450ab8807f5:33961 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14dd013, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0450ab8807f5/172.17.0.2:0 2024-11-16T12:51:33,355 DEBUG [RS:0;0450ab8807f5:32881 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48ac88c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0450ab8807f5/172.17.0.2:0 2024-11-16T12:51:33,365 DEBUG [RS:0;0450ab8807f5:32881 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0450ab8807f5:32881 2024-11-16T12:51:33,365 DEBUG [RS:1;0450ab8807f5:33961 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;0450ab8807f5:33961 2024-11-16T12:51:33,365 INFO [RS:0;0450ab8807f5:32881 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T12:51:33,365 INFO [RS:0;0450ab8807f5:32881 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T12:51:33,365 DEBUG [RS:0;0450ab8807f5:32881 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T12:51:33,365 INFO [RS:1;0450ab8807f5:33961 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T12:51:33,365 INFO [RS:1;0450ab8807f5:33961 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T12:51:33,365 DEBUG [RS:1;0450ab8807f5:33961 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T12:51:33,366 INFO [RS:0;0450ab8807f5:32881 {}] regionserver.HRegionServer(2659): reportForDuty to master=0450ab8807f5,40581,1731761492744 with port=32881, startcode=1731761492857 2024-11-16T12:51:33,366 INFO [RS:1;0450ab8807f5:33961 {}] regionserver.HRegionServer(2659): reportForDuty to master=0450ab8807f5,40581,1731761492744 with port=33961, startcode=1731761492894 2024-11-16T12:51:33,366 DEBUG [RS:1;0450ab8807f5:33961 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T12:51:33,366 DEBUG [RS:0;0450ab8807f5:32881 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T12:51:33,368 INFO [HMaster-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35365, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T12:51:33,368 INFO [HMaster-EventLoopGroup-15-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38891, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T12:51:33,369 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40581 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0450ab8807f5,32881,1731761492857 2024-11-16T12:51:33,369 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40581 {}] master.ServerManager(517): Registering regionserver=0450ab8807f5,32881,1731761492857 2024-11-16T12:51:33,371 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40581 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0450ab8807f5,33961,1731761492894 2024-11-16T12:51:33,371 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40581 {}] master.ServerManager(517): Registering regionserver=0450ab8807f5,33961,1731761492894 2024-11-16T12:51:33,373 DEBUG [RS:1;0450ab8807f5:33961 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:33,373 DEBUG [RS:1;0450ab8807f5:33961 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44975 2024-11-16T12:51:33,373 DEBUG [RS:1;0450ab8807f5:33961 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T12:51:33,378 DEBUG [RS:0;0450ab8807f5:32881 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:33,378 DEBUG [RS:0;0450ab8807f5:32881 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44975 2024-11-16T12:51:33,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40581-0x10144fc6ca60012, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T12:51:33,378 DEBUG [RS:0;0450ab8807f5:32881 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T12:51:33,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T12:51:33,387 DEBUG [RS:1;0450ab8807f5:33961 {}] zookeeper.ZKUtil(111): regionserver:33961-0x10144fc6ca60015, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0450ab8807f5,33961,1731761492894 2024-11-16T12:51:33,387 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0450ab8807f5,33961,1731761492894] 2024-11-16T12:51:33,387 WARN [RS:1;0450ab8807f5:33961 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T12:51:33,387 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0450ab8807f5,32881,1731761492857] 2024-11-16T12:51:33,387 INFO [RS:1;0450ab8807f5:33961 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:51:33,387 DEBUG [RS:0;0450ab8807f5:32881 {}] zookeeper.ZKUtil(111): regionserver:32881-0x10144fc6ca60014, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0450ab8807f5,32881,1731761492857 2024-11-16T12:51:33,387 DEBUG [RS:1;0450ab8807f5:33961 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,33961,1731761492894 2024-11-16T12:51:33,387 WARN [RS:0;0450ab8807f5:32881 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T12:51:33,387 INFO [RS:0;0450ab8807f5:32881 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:51:33,387 DEBUG [RS:0;0450ab8807f5:32881 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,32881,1731761492857 2024-11-16T12:51:33,390 INFO [RS:1;0450ab8807f5:33961 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T12:51:33,390 INFO [RS:0;0450ab8807f5:32881 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T12:51:33,393 INFO [RS:1;0450ab8807f5:33961 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T12:51:33,393 INFO [RS:1;0450ab8807f5:33961 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T12:51:33,393 INFO [RS:1;0450ab8807f5:33961 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:33,393 INFO [RS:1;0450ab8807f5:33961 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T12:51:33,394 INFO [RS:0;0450ab8807f5:32881 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T12:51:33,394 INFO [RS:0;0450ab8807f5:32881 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T12:51:33,394 INFO [RS:0;0450ab8807f5:32881 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:33,394 INFO [RS:0;0450ab8807f5:32881 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T12:51:33,395 INFO [RS:1;0450ab8807f5:33961 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T12:51:33,395 INFO [RS:1;0450ab8807f5:33961 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:33,395 INFO [RS:0;0450ab8807f5:32881 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T12:51:33,395 DEBUG [RS:1;0450ab8807f5:33961 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:33,395 INFO [RS:0;0450ab8807f5:32881 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:33,395 DEBUG [RS:1;0450ab8807f5:33961 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:33,395 DEBUG [RS:0;0450ab8807f5:32881 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:33,395 DEBUG [RS:1;0450ab8807f5:33961 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:33,395 DEBUG [RS:0;0450ab8807f5:32881 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:33,395 DEBUG [RS:1;0450ab8807f5:33961 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:33,395 DEBUG [RS:0;0450ab8807f5:32881 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:33,395 DEBUG [RS:1;0450ab8807f5:33961 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:33,395 DEBUG [RS:0;0450ab8807f5:32881 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:33,395 DEBUG [RS:0;0450ab8807f5:32881 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:33,395 DEBUG [RS:1;0450ab8807f5:33961 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0, corePoolSize=2, maxPoolSize=2 2024-11-16T12:51:33,395 DEBUG [RS:0;0450ab8807f5:32881 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0, corePoolSize=2, maxPoolSize=2 2024-11-16T12:51:33,395 DEBUG [RS:1;0450ab8807f5:33961 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:33,395 DEBUG [RS:0;0450ab8807f5:32881 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:33,396 DEBUG [RS:0;0450ab8807f5:32881 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:33,396 DEBUG [RS:1;0450ab8807f5:33961 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:33,396 DEBUG [RS:0;0450ab8807f5:32881 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:33,396 DEBUG [RS:1;0450ab8807f5:33961 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:33,396 DEBUG [RS:0;0450ab8807f5:32881 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:33,396 DEBUG [RS:1;0450ab8807f5:33961 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:33,396 DEBUG [RS:0;0450ab8807f5:32881 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:33,396 DEBUG [RS:1;0450ab8807f5:33961 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:33,396 DEBUG [RS:0;0450ab8807f5:32881 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:33,396 DEBUG [RS:1;0450ab8807f5:33961 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:33,396 DEBUG [RS:0;0450ab8807f5:32881 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:51:33,396 DEBUG [RS:1;0450ab8807f5:33961 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:51:33,396 DEBUG [RS:0;0450ab8807f5:32881 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:51:33,396 DEBUG [RS:1;0450ab8807f5:33961 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:51:33,403 INFO [RS:0;0450ab8807f5:32881 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:33,403 INFO [RS:1;0450ab8807f5:33961 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:33,403 INFO [RS:0;0450ab8807f5:32881 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:33,403 INFO [RS:1;0450ab8807f5:33961 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:33,403 INFO [RS:0;0450ab8807f5:32881 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:33,403 INFO [RS:1;0450ab8807f5:33961 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:33,403 INFO [RS:0;0450ab8807f5:32881 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:33,403 INFO [RS:1;0450ab8807f5:33961 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:33,403 INFO [RS:1;0450ab8807f5:33961 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:33,403 INFO [RS:0;0450ab8807f5:32881 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:33,403 INFO [RS:1;0450ab8807f5:33961 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,33961,1731761492894-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T12:51:33,403 INFO [RS:0;0450ab8807f5:32881 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,32881,1731761492857-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T12:51:33,416 INFO [RS:1;0450ab8807f5:33961 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T12:51:33,417 INFO [RS:1;0450ab8807f5:33961 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,33961,1731761492894-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:33,417 INFO [RS:1;0450ab8807f5:33961 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:33,417 INFO [RS:1;0450ab8807f5:33961 {}] regionserver.Replication(171): 0450ab8807f5,33961,1731761492894 started 2024-11-16T12:51:33,420 INFO [RS:0;0450ab8807f5:32881 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T12:51:33,421 INFO [RS:0;0450ab8807f5:32881 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,32881,1731761492857-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:33,421 INFO [RS:0;0450ab8807f5:32881 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:33,421 INFO [RS:0;0450ab8807f5:32881 {}] regionserver.Replication(171): 0450ab8807f5,32881,1731761492857 started 2024-11-16T12:51:33,430 INFO [RS:1;0450ab8807f5:33961 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:33,431 INFO [RS:1;0450ab8807f5:33961 {}] regionserver.HRegionServer(1482): Serving as 0450ab8807f5,33961,1731761492894, RpcServer on 0450ab8807f5/172.17.0.2:33961, sessionid=0x10144fc6ca60015 2024-11-16T12:51:33,431 DEBUG [RS:1;0450ab8807f5:33961 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T12:51:33,431 DEBUG [RS:1;0450ab8807f5:33961 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0450ab8807f5,33961,1731761492894 2024-11-16T12:51:33,431 DEBUG [RS:1;0450ab8807f5:33961 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,33961,1731761492894' 2024-11-16T12:51:33,431 DEBUG [RS:1;0450ab8807f5:33961 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T12:51:33,431 DEBUG [RS:1;0450ab8807f5:33961 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T12:51:33,432 DEBUG [RS:1;0450ab8807f5:33961 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T12:51:33,432 DEBUG [RS:1;0450ab8807f5:33961 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T12:51:33,432 DEBUG [RS:1;0450ab8807f5:33961 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0450ab8807f5,33961,1731761492894 2024-11-16T12:51:33,432 DEBUG [RS:1;0450ab8807f5:33961 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,33961,1731761492894' 2024-11-16T12:51:33,432 DEBUG [RS:1;0450ab8807f5:33961 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T12:51:33,432 DEBUG [RS:1;0450ab8807f5:33961 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T12:51:33,433 DEBUG [RS:1;0450ab8807f5:33961 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T12:51:33,433 INFO [RS:1;0450ab8807f5:33961 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T12:51:33,433 INFO [RS:1;0450ab8807f5:33961 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T12:51:33,436 INFO [RS:0;0450ab8807f5:32881 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:33,437 INFO [RS:0;0450ab8807f5:32881 {}] regionserver.HRegionServer(1482): Serving as 0450ab8807f5,32881,1731761492857, RpcServer on 0450ab8807f5/172.17.0.2:32881, sessionid=0x10144fc6ca60014 2024-11-16T12:51:33,437 DEBUG [RS:0;0450ab8807f5:32881 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T12:51:33,437 DEBUG [RS:0;0450ab8807f5:32881 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0450ab8807f5,32881,1731761492857 2024-11-16T12:51:33,437 DEBUG [RS:0;0450ab8807f5:32881 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,32881,1731761492857' 2024-11-16T12:51:33,437 DEBUG [RS:0;0450ab8807f5:32881 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T12:51:33,437 DEBUG [RS:0;0450ab8807f5:32881 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T12:51:33,438 DEBUG [RS:0;0450ab8807f5:32881 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T12:51:33,438 DEBUG [RS:0;0450ab8807f5:32881 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T12:51:33,438 DEBUG [RS:0;0450ab8807f5:32881 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0450ab8807f5,32881,1731761492857 2024-11-16T12:51:33,438 DEBUG [RS:0;0450ab8807f5:32881 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,32881,1731761492857' 2024-11-16T12:51:33,438 DEBUG [RS:0;0450ab8807f5:32881 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T12:51:33,438 DEBUG [RS:0;0450ab8807f5:32881 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T12:51:33,438 DEBUG [RS:0;0450ab8807f5:32881 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T12:51:33,438 INFO [RS:0;0450ab8807f5:32881 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T12:51:33,438 INFO [RS:0;0450ab8807f5:32881 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T12:51:33,465 WARN [0450ab8807f5:40581 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-16T12:51:33,533 INFO [RS:1;0450ab8807f5:33961 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T12:51:33,535 INFO [RS:1;0450ab8807f5:33961 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0450ab8807f5%2C33961%2C1731761492894, suffix=, logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,33961,1731761492894, archiveDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/oldWALs, maxLogs=32 2024-11-16T12:51:33,539 INFO [RS:0;0450ab8807f5:32881 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T12:51:33,540 INFO [RS:0;0450ab8807f5:32881 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0450ab8807f5%2C32881%2C1731761492857, suffix=, logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,32881,1731761492857, archiveDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/oldWALs, maxLogs=32 2024-11-16T12:51:33,547 DEBUG [RS:1;0450ab8807f5:33961 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,33961,1731761492894/0450ab8807f5%2C33961%2C1731761492894.1731761493535, exclude list is [], retry=0 2024-11-16T12:51:33,550 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36961,DS-7ef9e9a6-250e-467c-989b-a9ffcab2299e,DISK] 2024-11-16T12:51:33,550 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43257,DS-06d91055-02a4-40d7-bddb-15aee81f02ac,DISK] 2024-11-16T12:51:33,552 INFO [RS:1;0450ab8807f5:33961 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,33961,1731761492894/0450ab8807f5%2C33961%2C1731761492894.1731761493535 2024-11-16T12:51:33,552 DEBUG [RS:1;0450ab8807f5:33961 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:46667:46667)] 2024-11-16T12:51:33,553 DEBUG [RS:0;0450ab8807f5:32881 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,32881,1731761492857/0450ab8807f5%2C32881%2C1731761492857.1731761493541, exclude list is [], retry=0 2024-11-16T12:51:33,556 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43257,DS-06d91055-02a4-40d7-bddb-15aee81f02ac,DISK] 2024-11-16T12:51:33,556 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36961,DS-7ef9e9a6-250e-467c-989b-a9ffcab2299e,DISK] 2024-11-16T12:51:33,563 INFO [RS:0;0450ab8807f5:32881 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,32881,1731761492857/0450ab8807f5%2C32881%2C1731761492857.1731761493541 2024-11-16T12:51:33,563 DEBUG [RS:0;0450ab8807f5:32881 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46667:46667),(127.0.0.1/127.0.0.1:38819:38819)] 2024-11-16T12:51:33,590 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T12:51:33,715 DEBUG [0450ab8807f5:40581 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-16T12:51:33,715 DEBUG [0450ab8807f5:40581 {}] balancer.BalancerClusterState(204): Hosts are {0450ab8807f5=0} racks are {/default-rack=0} 2024-11-16T12:51:33,717 DEBUG [0450ab8807f5:40581 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T12:51:33,717 DEBUG [0450ab8807f5:40581 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T12:51:33,717 DEBUG [0450ab8807f5:40581 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T12:51:33,717 DEBUG [0450ab8807f5:40581 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T12:51:33,717 INFO [0450ab8807f5:40581 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T12:51:33,717 INFO [0450ab8807f5:40581 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T12:51:33,717 DEBUG [0450ab8807f5:40581 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T12:51:33,718 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0450ab8807f5,32881,1731761492857 2024-11-16T12:51:33,719 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0450ab8807f5,32881,1731761492857, state=OPENING 2024-11-16T12:51:33,745 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-16T12:51:33,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32881-0x10144fc6ca60014, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:33,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40581-0x10144fc6ca60012, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:33,753 DEBUG [pool-593-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10144fc6ca60015, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:33,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:33,754 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T12:51:33,754 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:33,754 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:33,754 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:33,754 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:33,754 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0450ab8807f5,32881,1731761492857}] 2024-11-16T12:51:33,908 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T12:51:33,909 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60739, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T12:51:33,912 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T12:51:33,912 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:51:33,913 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T12:51:33,914 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0450ab8807f5%2C32881%2C1731761492857.meta, suffix=.meta, logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,32881,1731761492857, archiveDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/oldWALs, maxLogs=32 2024-11-16T12:51:33,926 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,32881,1731761492857/0450ab8807f5%2C32881%2C1731761492857.meta.1731761493915.meta, exclude list is [], retry=0 2024-11-16T12:51:33,928 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43257,DS-06d91055-02a4-40d7-bddb-15aee81f02ac,DISK] 2024-11-16T12:51:33,928 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36961,DS-7ef9e9a6-250e-467c-989b-a9ffcab2299e,DISK] 2024-11-16T12:51:33,930 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,32881,1731761492857/0450ab8807f5%2C32881%2C1731761492857.meta.1731761493915.meta 2024-11-16T12:51:33,931 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46667:46667),(127.0.0.1/127.0.0.1:38819:38819)] 2024-11-16T12:51:33,931 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T12:51:33,931 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T12:51:33,931 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T12:51:33,931 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T12:51:33,931 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T12:51:33,931 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:33,931 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T12:51:33,931 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T12:51:33,933 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T12:51:33,934 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T12:51:33,934 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:33,934 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:33,934 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T12:51:33,935 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T12:51:33,935 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:33,936 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:33,936 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T12:51:33,937 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T12:51:33,937 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:33,937 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:33,937 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T12:51:33,938 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T12:51:33,938 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:33,938 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:33,938 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T12:51:33,939 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740 2024-11-16T12:51:33,940 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740 2024-11-16T12:51:33,941 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T12:51:33,941 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T12:51:33,942 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T12:51:33,943 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T12:51:33,944 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60964371, jitterRate=-0.09156008064746857}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T12:51:33,944 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T12:51:33,945 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731761493932Writing region info on filesystem at 1731761493932Initializing all the Stores at 1731761493933 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761493933Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761493933Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761493933Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761493933Cleaning up temporary data from old regions at 1731761493941 (+8 ms)Running coprocessor post-open hooks at 1731761493944 (+3 ms)Region opened successfully at 1731761493944 2024-11-16T12:51:33,945 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731761493907 2024-11-16T12:51:33,948 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T12:51:33,948 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T12:51:33,949 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0450ab8807f5,32881,1731761492857 2024-11-16T12:51:33,950 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0450ab8807f5,32881,1731761492857, state=OPEN 2024-11-16T12:51:33,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32881-0x10144fc6ca60014, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:33,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40581-0x10144fc6ca60012, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:33,962 DEBUG [pool-593-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10144fc6ca60015, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:33,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:33,962 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:33,962 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:33,962 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:33,962 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:33,962 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0450ab8807f5,32881,1731761492857 2024-11-16T12:51:33,965 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-16T12:51:33,965 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0450ab8807f5,32881,1731761492857 in 208 msec 2024-11-16T12:51:33,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-16T12:51:33,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 653 msec 2024-11-16T12:51:33,968 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T12:51:33,968 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-16T12:51:33,969 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T12:51:33,969 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,32881,1731761492857, seqNum=-1] 2024-11-16T12:51:33,970 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T12:51:33,971 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34907, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T12:51:33,976 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 710 msec 2024-11-16T12:51:33,976 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731761493976, completionTime=-1 2024-11-16T12:51:33,976 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-16T12:51:33,976 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T12:51:33,978 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T12:51:33,978 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731761553978 2024-11-16T12:51:33,978 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731761613978 2024-11-16T12:51:33,979 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-16T12:51:33,979 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-16T12:51:33,979 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,40581,1731761492744-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:33,979 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,40581,1731761492744-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:33,979 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,40581,1731761492744-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:33,979 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0450ab8807f5:40581, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:33,979 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:33,980 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:33,982 DEBUG [master/0450ab8807f5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-16T12:51:33,984 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.005sec 2024-11-16T12:51:33,985 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-16T12:51:33,985 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-16T12:51:33,985 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-16T12:51:33,985 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-16T12:51:33,985 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-16T12:51:33,985 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,40581,1731761492744-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T12:51:33,985 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,40581,1731761492744-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-16T12:51:33,987 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-16T12:51:33,988 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-16T12:51:33,988 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,40581,1731761492744-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:34,025 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x693ac9cf to 127.0.0.1:53208 with session timeout=1000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@34c87d19 2024-11-16T12:51:34,062 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@662a5c1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T12:51:34,063 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T12:51:34,064 DEBUG [ReadOnlyZKClient-127.0.0.1:53208@0x693ac9cf {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,32881,1731761492857, seqNum=-1] 2024-11-16T12:51:34,064 DEBUG [ReadOnlyZKClient-127.0.0.1:53208@0x693ac9cf {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T12:51:34,066 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33650, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T12:51:34,068 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0450ab8807f5,40581,1731761492744 2024-11-16T12:51:34,088 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0450ab8807f5:0 server-side Connection retries=45 2024-11-16T12:51:34,088 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:34,088 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:34,088 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T12:51:34,088 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:34,088 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T12:51:34,088 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T12:51:34,088 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T12:51:34,089 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39017 2024-11-16T12:51:34,090 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39017 connecting to ZooKeeper ensemble=127.0.0.1:53208 2024-11-16T12:51:34,091 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:34,093 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:34,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:390170x0, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:51:34,104 DEBUG [pool-713-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-16T12:51:34,104 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39017-0x10144fc6ca60017 connected 2024-11-16T12:51:34,104 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:39017-0x10144fc6ca60017, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-16T12:51:34,105 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T12:51:34,105 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T12:51:34,105 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:39017-0x10144fc6ca60017, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T12:51:34,107 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39017-0x10144fc6ca60017, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T12:51:34,108 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39017 2024-11-16T12:51:34,108 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39017 2024-11-16T12:51:34,108 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39017 2024-11-16T12:51:34,109 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39017 2024-11-16T12:51:34,109 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39017 2024-11-16T12:51:34,110 INFO [RS:2;0450ab8807f5:39017 {}] regionserver.HRegionServer(746): ClusterId : 275eeb35-c30e-47fc-a490-ffb4d908b232 2024-11-16T12:51:34,110 DEBUG [RS:2;0450ab8807f5:39017 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T12:51:34,121 DEBUG [RS:2;0450ab8807f5:39017 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T12:51:34,121 DEBUG [RS:2;0450ab8807f5:39017 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T12:51:34,129 DEBUG [RS:2;0450ab8807f5:39017 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T12:51:34,130 DEBUG [RS:2;0450ab8807f5:39017 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c5301a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0450ab8807f5/172.17.0.2:0 2024-11-16T12:51:34,141 DEBUG [RS:2;0450ab8807f5:39017 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;0450ab8807f5:39017 2024-11-16T12:51:34,141 INFO [RS:2;0450ab8807f5:39017 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T12:51:34,141 INFO [RS:2;0450ab8807f5:39017 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T12:51:34,141 DEBUG [RS:2;0450ab8807f5:39017 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T12:51:34,142 INFO [RS:2;0450ab8807f5:39017 {}] regionserver.HRegionServer(2659): reportForDuty to master=0450ab8807f5,40581,1731761492744 with port=39017, startcode=1731761494087 2024-11-16T12:51:34,142 DEBUG [RS:2;0450ab8807f5:39017 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T12:51:34,144 INFO [HMaster-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35639, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.8 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T12:51:34,144 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40581 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0450ab8807f5,39017,1731761494087 2024-11-16T12:51:34,144 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40581 {}] master.ServerManager(517): Registering regionserver=0450ab8807f5,39017,1731761494087 2024-11-16T12:51:34,146 DEBUG [RS:2;0450ab8807f5:39017 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:34,146 DEBUG [RS:2;0450ab8807f5:39017 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44975 2024-11-16T12:51:34,146 DEBUG [RS:2;0450ab8807f5:39017 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T12:51:34,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T12:51:34,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40581-0x10144fc6ca60012, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T12:51:34,154 DEBUG [RS:2;0450ab8807f5:39017 {}] zookeeper.ZKUtil(111): regionserver:39017-0x10144fc6ca60017, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0450ab8807f5,39017,1731761494087 2024-11-16T12:51:34,154 WARN [RS:2;0450ab8807f5:39017 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T12:51:34,154 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0450ab8807f5,39017,1731761494087] 2024-11-16T12:51:34,154 INFO [RS:2;0450ab8807f5:39017 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:51:34,154 DEBUG [RS:2;0450ab8807f5:39017 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,39017,1731761494087 2024-11-16T12:51:34,158 INFO [RS:2;0450ab8807f5:39017 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T12:51:34,159 INFO [RS:2;0450ab8807f5:39017 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T12:51:34,160 INFO [RS:2;0450ab8807f5:39017 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T12:51:34,160 INFO [RS:2;0450ab8807f5:39017 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:34,160 INFO [RS:2;0450ab8807f5:39017 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T12:51:34,161 INFO [RS:2;0450ab8807f5:39017 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T12:51:34,161 INFO [RS:2;0450ab8807f5:39017 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:34,161 DEBUG [RS:2;0450ab8807f5:39017 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:34,161 DEBUG [RS:2;0450ab8807f5:39017 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:34,161 DEBUG [RS:2;0450ab8807f5:39017 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:34,161 DEBUG [RS:2;0450ab8807f5:39017 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:34,161 DEBUG [RS:2;0450ab8807f5:39017 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:34,161 DEBUG [RS:2;0450ab8807f5:39017 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0, corePoolSize=2, maxPoolSize=2 2024-11-16T12:51:34,161 DEBUG [RS:2;0450ab8807f5:39017 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:34,161 DEBUG [RS:2;0450ab8807f5:39017 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:34,161 DEBUG [RS:2;0450ab8807f5:39017 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:34,161 DEBUG [RS:2;0450ab8807f5:39017 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:34,161 DEBUG [RS:2;0450ab8807f5:39017 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:34,161 DEBUG [RS:2;0450ab8807f5:39017 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:34,162 DEBUG [RS:2;0450ab8807f5:39017 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:51:34,162 DEBUG [RS:2;0450ab8807f5:39017 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:51:34,162 INFO [RS:2;0450ab8807f5:39017 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:34,162 INFO [RS:2;0450ab8807f5:39017 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:34,162 INFO [RS:2;0450ab8807f5:39017 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:34,162 INFO [RS:2;0450ab8807f5:39017 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:34,162 INFO [RS:2;0450ab8807f5:39017 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:34,162 INFO [RS:2;0450ab8807f5:39017 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,39017,1731761494087-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T12:51:34,176 INFO [RS:2;0450ab8807f5:39017 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T12:51:34,176 INFO [RS:2;0450ab8807f5:39017 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,39017,1731761494087-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:34,176 INFO [RS:2;0450ab8807f5:39017 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:34,177 INFO [RS:2;0450ab8807f5:39017 {}] regionserver.Replication(171): 0450ab8807f5,39017,1731761494087 started 2024-11-16T12:51:34,190 INFO [RS:2;0450ab8807f5:39017 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:34,190 INFO [RS:2;0450ab8807f5:39017 {}] regionserver.HRegionServer(1482): Serving as 0450ab8807f5,39017,1731761494087, RpcServer on 0450ab8807f5/172.17.0.2:39017, sessionid=0x10144fc6ca60017 2024-11-16T12:51:34,190 DEBUG [RS:2;0450ab8807f5:39017 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T12:51:34,190 DEBUG [RS:2;0450ab8807f5:39017 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0450ab8807f5,39017,1731761494087 2024-11-16T12:51:34,190 DEBUG [RS:2;0450ab8807f5:39017 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,39017,1731761494087' 2024-11-16T12:51:34,190 DEBUG [RS:2;0450ab8807f5:39017 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T12:51:34,191 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T12:51:34,191 DEBUG [RS:2;0450ab8807f5:39017 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T12:51:34,192 DEBUG [RS:2;0450ab8807f5:39017 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T12:51:34,192 DEBUG [RS:2;0450ab8807f5:39017 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T12:51:34,192 DEBUG [ReadOnlyZKClient-127.0.0.1:53208@0x693ac9cf {}] client.AsyncConnectionImpl(321): The fetched master address is 0450ab8807f5,40581,1731761492744 2024-11-16T12:51:34,192 DEBUG [RS:2;0450ab8807f5:39017 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0450ab8807f5,39017,1731761494087 2024-11-16T12:51:34,192 DEBUG [RS:2;0450ab8807f5:39017 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,39017,1731761494087' 2024-11-16T12:51:34,192 DEBUG [ReadOnlyZKClient-127.0.0.1:53208@0x693ac9cf {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2cb13612 2024-11-16T12:51:34,192 DEBUG [RS:2;0450ab8807f5:39017 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T12:51:34,192 DEBUG [ReadOnlyZKClient-127.0.0.1:53208@0x693ac9cf {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T12:51:34,192 DEBUG [RS:2;0450ab8807f5:39017 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T12:51:34,192 DEBUG [RS:2;0450ab8807f5:39017 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T12:51:34,192 INFO [RS:2;0450ab8807f5:39017 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T12:51:34,192 INFO [RS:2;0450ab8807f5:39017 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T12:51:34,194 INFO [HMaster-EventLoopGroup-15-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60588, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T12:51:34,195 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40581 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testLogSplittingAfterMasterRecoveryDueToZKExpiry', {NAME => 'col', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T12:51:34,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40581 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry 2024-11-16T12:51:34,197 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T12:51:34,198 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:34,198 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40581 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testLogSplittingAfterMasterRecoveryDueToZKExpiry" procId is: 4 2024-11-16T12:51:34,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40581 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T12:51:34,199 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T12:51:34,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741893_1071 (size=399) 2024-11-16T12:51:34,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741893_1071 (size=399) 2024-11-16T12:51:34,209 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 75073183d2c44df3f804b5dab504292d, NAME => 'testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1731761494194.75073183d2c44df3f804b5dab504292d.', STARTKEY => '2', ENDKEY => '3'}, tableDescriptor='testLogSplittingAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'col', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:34,210 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => ef3a398c9e5522ea013cb58384c3d2e5, NAME => 'testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1731761494194.ef3a398c9e5522ea013cb58384c3d2e5.', STARTKEY => '1', ENDKEY => '2'}, tableDescriptor='testLogSplittingAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'col', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:34,210 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9c4c0c2626292bc7607edf03a64b4c11, NAME => 'testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11.', STARTKEY => '', ENDKEY => '1'}, tableDescriptor='testLogSplittingAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'col', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:34,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741894_1072 (size=85) 2024-11-16T12:51:34,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741894_1072 (size=85) 2024-11-16T12:51:34,220 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(898): Instantiated testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1731761494194.ef3a398c9e5522ea013cb58384c3d2e5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:34,220 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1722): Closing ef3a398c9e5522ea013cb58384c3d2e5, disabling compactions & flushes 2024-11-16T12:51:34,220 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1755): Closing region testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1731761494194.ef3a398c9e5522ea013cb58384c3d2e5. 2024-11-16T12:51:34,220 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1731761494194.ef3a398c9e5522ea013cb58384c3d2e5. 2024-11-16T12:51:34,220 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1731761494194.ef3a398c9e5522ea013cb58384c3d2e5. after waiting 0 ms 2024-11-16T12:51:34,221 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1731761494194.ef3a398c9e5522ea013cb58384c3d2e5. 2024-11-16T12:51:34,221 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1973): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1731761494194.ef3a398c9e5522ea013cb58384c3d2e5. 2024-11-16T12:51:34,221 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1676): Region close journal for ef3a398c9e5522ea013cb58384c3d2e5: Waiting for close lock at 1731761494220Disabling compacts and flushes for region at 1731761494220Disabling writes for close at 1731761494220Writing region close event to WAL at 1731761494221 (+1 ms)Closed at 1731761494221 2024-11-16T12:51:34,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741895_1073 (size=85) 2024-11-16T12:51:34,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741895_1073 (size=85) 2024-11-16T12:51:34,222 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 4432ea1606b52a5734457a3cb3caaee3, NAME => 'testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1731761494194.4432ea1606b52a5734457a3cb3caaee3.', STARTKEY => '3', ENDKEY => '4'}, tableDescriptor='testLogSplittingAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'col', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:34,222 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(898): Instantiated testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1731761494194.75073183d2c44df3f804b5dab504292d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:34,222 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1722): Closing 75073183d2c44df3f804b5dab504292d, disabling compactions & flushes 2024-11-16T12:51:34,222 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1755): Closing region testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1731761494194.75073183d2c44df3f804b5dab504292d. 2024-11-16T12:51:34,222 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1731761494194.75073183d2c44df3f804b5dab504292d. 2024-11-16T12:51:34,222 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1731761494194.75073183d2c44df3f804b5dab504292d. after waiting 0 ms 2024-11-16T12:51:34,222 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1731761494194.75073183d2c44df3f804b5dab504292d. 2024-11-16T12:51:34,222 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1973): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1731761494194.75073183d2c44df3f804b5dab504292d. 2024-11-16T12:51:34,222 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1676): Region close journal for 75073183d2c44df3f804b5dab504292d: Waiting for close lock at 1731761494222Disabling compacts and flushes for region at 1731761494222Disabling writes for close at 1731761494222Writing region close event to WAL at 1731761494222Closed at 1731761494222 2024-11-16T12:51:34,223 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => e39334374ca56418de9e14f727ef8576, NAME => 'testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1731761494194.e39334374ca56418de9e14f727ef8576.', STARTKEY => '4', ENDKEY => '5'}, tableDescriptor='testLogSplittingAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'col', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:34,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741896_1074 (size=84) 2024-11-16T12:51:34,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741896_1074 (size=84) 2024-11-16T12:51:34,228 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(898): Instantiated testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:34,228 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1722): Closing 9c4c0c2626292bc7607edf03a64b4c11, disabling compactions & flushes 2024-11-16T12:51:34,228 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1755): Closing region testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11. 2024-11-16T12:51:34,228 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11. 2024-11-16T12:51:34,228 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11. after waiting 0 ms 2024-11-16T12:51:34,228 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11. 2024-11-16T12:51:34,228 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1973): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11. 2024-11-16T12:51:34,228 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9c4c0c2626292bc7607edf03a64b4c11: Waiting for close lock at 1731761494228Disabling compacts and flushes for region at 1731761494228Disabling writes for close at 1731761494228Writing region close event to WAL at 1731761494228Closed at 1731761494228 2024-11-16T12:51:34,229 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8d3e8566a59cc6153f2f93cef2a417aa, NAME => 'testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1731761494194.8d3e8566a59cc6153f2f93cef2a417aa.', STARTKEY => '5', ENDKEY => ''}, tableDescriptor='testLogSplittingAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'col', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:34,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741897_1075 (size=85) 2024-11-16T12:51:34,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741898_1076 (size=85) 2024-11-16T12:51:34,231 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(898): Instantiated testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1731761494194.4432ea1606b52a5734457a3cb3caaee3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:34,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741898_1076 (size=85) 2024-11-16T12:51:34,231 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1722): Closing 4432ea1606b52a5734457a3cb3caaee3, disabling compactions & flushes 2024-11-16T12:51:34,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741897_1075 (size=85) 2024-11-16T12:51:34,231 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1755): Closing region testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1731761494194.4432ea1606b52a5734457a3cb3caaee3. 2024-11-16T12:51:34,231 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1731761494194.4432ea1606b52a5734457a3cb3caaee3. 2024-11-16T12:51:34,231 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1731761494194.4432ea1606b52a5734457a3cb3caaee3. after waiting 0 ms 2024-11-16T12:51:34,231 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1731761494194.4432ea1606b52a5734457a3cb3caaee3. 2024-11-16T12:51:34,231 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1973): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1731761494194.4432ea1606b52a5734457a3cb3caaee3. 2024-11-16T12:51:34,231 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1676): Region close journal for 4432ea1606b52a5734457a3cb3caaee3: Waiting for close lock at 1731761494231Disabling compacts and flushes for region at 1731761494231Disabling writes for close at 1731761494231Writing region close event to WAL at 1731761494231Closed at 1731761494231 2024-11-16T12:51:34,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741899_1077 (size=84) 2024-11-16T12:51:34,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741899_1077 (size=84) 2024-11-16T12:51:34,237 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(898): Instantiated testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1731761494194.8d3e8566a59cc6153f2f93cef2a417aa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:34,237 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1722): Closing 8d3e8566a59cc6153f2f93cef2a417aa, disabling compactions & flushes 2024-11-16T12:51:34,237 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1755): Closing region testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1731761494194.8d3e8566a59cc6153f2f93cef2a417aa. 2024-11-16T12:51:34,237 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1731761494194.8d3e8566a59cc6153f2f93cef2a417aa. 2024-11-16T12:51:34,237 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1731761494194.8d3e8566a59cc6153f2f93cef2a417aa. after waiting 0 ms 2024-11-16T12:51:34,237 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1731761494194.8d3e8566a59cc6153f2f93cef2a417aa. 2024-11-16T12:51:34,237 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1973): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1731761494194.8d3e8566a59cc6153f2f93cef2a417aa. 2024-11-16T12:51:34,237 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8d3e8566a59cc6153f2f93cef2a417aa: Waiting for close lock at 1731761494237Disabling compacts and flushes for region at 1731761494237Disabling writes for close at 1731761494237Writing region close event to WAL at 1731761494237Closed at 1731761494237 2024-11-16T12:51:34,293 INFO [RS:2;0450ab8807f5:39017 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T12:51:34,294 INFO [RS:2;0450ab8807f5:39017 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0450ab8807f5%2C39017%2C1731761494087, suffix=, logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,39017,1731761494087, archiveDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/oldWALs, maxLogs=32 2024-11-16T12:51:34,307 DEBUG [RS:2;0450ab8807f5:39017 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,39017,1731761494087/0450ab8807f5%2C39017%2C1731761494087.1731761494295, exclude list is [], retry=0 2024-11-16T12:51:34,309 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-19-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36961,DS-7ef9e9a6-250e-467c-989b-a9ffcab2299e,DISK] 2024-11-16T12:51:34,309 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-19-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43257,DS-06d91055-02a4-40d7-bddb-15aee81f02ac,DISK] 2024-11-16T12:51:34,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40581 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T12:51:34,311 INFO [RS:2;0450ab8807f5:39017 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,39017,1731761494087/0450ab8807f5%2C39017%2C1731761494087.1731761494295 2024-11-16T12:51:34,312 DEBUG [RS:2;0450ab8807f5:39017 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46667:46667),(127.0.0.1/127.0.0.1:38819:38819)] 2024-11-16T12:51:34,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40581 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T12:51:34,631 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(898): Instantiated testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1731761494194.e39334374ca56418de9e14f727ef8576.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:34,632 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1722): Closing e39334374ca56418de9e14f727ef8576, disabling compactions & flushes 2024-11-16T12:51:34,632 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1755): Closing region testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1731761494194.e39334374ca56418de9e14f727ef8576. 2024-11-16T12:51:34,632 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1731761494194.e39334374ca56418de9e14f727ef8576. 2024-11-16T12:51:34,632 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1731761494194.e39334374ca56418de9e14f727ef8576. after waiting 0 ms 2024-11-16T12:51:34,632 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1731761494194.e39334374ca56418de9e14f727ef8576. 2024-11-16T12:51:34,632 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1973): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1731761494194.e39334374ca56418de9e14f727ef8576. 2024-11-16T12:51:34,632 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1676): Region close journal for e39334374ca56418de9e14f727ef8576: Waiting for close lock at 1731761494632Disabling compacts and flushes for region at 1731761494632Disabling writes for close at 1731761494632Writing region close event to WAL at 1731761494632Closed at 1731761494632 2024-11-16T12:51:34,633 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T12:51:34,634 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1731761494194.ef3a398c9e5522ea013cb58384c3d2e5.","families":{"info":[{"qualifier":"regioninfo","vlen":84,"tag":[],"timestamp":"1731761494633"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731761494633"}]},"ts":"1731761494633"} 2024-11-16T12:51:34,634 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1731761494194.75073183d2c44df3f804b5dab504292d.","families":{"info":[{"qualifier":"regioninfo","vlen":84,"tag":[],"timestamp":"1731761494633"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731761494633"}]},"ts":"1731761494633"} 2024-11-16T12:51:34,634 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11.","families":{"info":[{"qualifier":"regioninfo","vlen":83,"tag":[],"timestamp":"1731761494633"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731761494633"}]},"ts":"1731761494633"} 2024-11-16T12:51:34,634 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1731761494194.4432ea1606b52a5734457a3cb3caaee3.","families":{"info":[{"qualifier":"regioninfo","vlen":84,"tag":[],"timestamp":"1731761494633"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731761494633"}]},"ts":"1731761494633"} 2024-11-16T12:51:34,634 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1731761494194.8d3e8566a59cc6153f2f93cef2a417aa.","families":{"info":[{"qualifier":"regioninfo","vlen":83,"tag":[],"timestamp":"1731761494633"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731761494633"}]},"ts":"1731761494633"} 2024-11-16T12:51:34,634 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1731761494194.e39334374ca56418de9e14f727ef8576.","families":{"info":[{"qualifier":"regioninfo","vlen":84,"tag":[],"timestamp":"1731761494633"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731761494633"}]},"ts":"1731761494633"} 2024-11-16T12:51:34,662 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 6 regions to meta. 2024-11-16T12:51:34,663 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T12:51:34,664 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testLogSplittingAfterMasterRecoveryDueToZKExpiry","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731761494663"}]},"ts":"1731761494663"} 2024-11-16T12:51:34,666 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testLogSplittingAfterMasterRecoveryDueToZKExpiry, state=ENABLING in hbase:meta 2024-11-16T12:51:34,666 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {0450ab8807f5=0} racks are {/default-rack=0} 2024-11-16T12:51:34,670 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T12:51:34,670 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T12:51:34,670 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-16T12:51:34,670 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T12:51:34,670 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T12:51:34,670 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-16T12:51:34,670 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T12:51:34,670 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T12:51:34,670 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-16T12:51:34,670 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T12:51:34,670 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=9c4c0c2626292bc7607edf03a64b4c11, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=ef3a398c9e5522ea013cb58384c3d2e5, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=75073183d2c44df3f804b5dab504292d, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=4432ea1606b52a5734457a3cb3caaee3, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=e39334374ca56418de9e14f727ef8576, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=8d3e8566a59cc6153f2f93cef2a417aa, ASSIGN}] 2024-11-16T12:51:34,672 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=e39334374ca56418de9e14f727ef8576, ASSIGN 2024-11-16T12:51:34,672 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=ef3a398c9e5522ea013cb58384c3d2e5, ASSIGN 2024-11-16T12:51:34,673 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=4432ea1606b52a5734457a3cb3caaee3, ASSIGN 2024-11-16T12:51:34,673 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=75073183d2c44df3f804b5dab504292d, ASSIGN 2024-11-16T12:51:34,674 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=75073183d2c44df3f804b5dab504292d, ASSIGN; state=OFFLINE, location=0450ab8807f5,32881,1731761492857; forceNewPlan=false, retain=false 2024-11-16T12:51:34,674 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=ef3a398c9e5522ea013cb58384c3d2e5, ASSIGN; state=OFFLINE, location=0450ab8807f5,33961,1731761492894; forceNewPlan=false, retain=false 2024-11-16T12:51:34,674 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=8d3e8566a59cc6153f2f93cef2a417aa, ASSIGN 2024-11-16T12:51:34,674 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=4432ea1606b52a5734457a3cb3caaee3, ASSIGN; state=OFFLINE, location=0450ab8807f5,33961,1731761492894; forceNewPlan=false, retain=false 2024-11-16T12:51:34,674 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=e39334374ca56418de9e14f727ef8576, ASSIGN; state=OFFLINE, location=0450ab8807f5,39017,1731761494087; forceNewPlan=false, retain=false 2024-11-16T12:51:34,675 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=8d3e8566a59cc6153f2f93cef2a417aa, ASSIGN; state=OFFLINE, location=0450ab8807f5,32881,1731761492857; forceNewPlan=false, retain=false 2024-11-16T12:51:34,678 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=9c4c0c2626292bc7607edf03a64b4c11, ASSIGN 2024-11-16T12:51:34,679 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=9c4c0c2626292bc7607edf03a64b4c11, ASSIGN; state=OFFLINE, location=0450ab8807f5,39017,1731761494087; forceNewPlan=false, retain=false 2024-11-16T12:51:34,824 INFO [0450ab8807f5:40581 {}] balancer.BaseLoadBalancer(388): Reassigned 6 regions. 6 retained the pre-restart assignment. 2024-11-16T12:51:34,825 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=4432ea1606b52a5734457a3cb3caaee3, regionState=OPENING, regionLocation=0450ab8807f5,33961,1731761492894 2024-11-16T12:51:34,825 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=ef3a398c9e5522ea013cb58384c3d2e5, regionState=OPENING, regionLocation=0450ab8807f5,33961,1731761492894 2024-11-16T12:51:34,825 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=8d3e8566a59cc6153f2f93cef2a417aa, regionState=OPENING, regionLocation=0450ab8807f5,32881,1731761492857 2024-11-16T12:51:34,825 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9c4c0c2626292bc7607edf03a64b4c11, regionState=OPENING, regionLocation=0450ab8807f5,39017,1731761494087 2024-11-16T12:51:34,825 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=e39334374ca56418de9e14f727ef8576, regionState=OPENING, regionLocation=0450ab8807f5,39017,1731761494087 2024-11-16T12:51:34,825 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=75073183d2c44df3f804b5dab504292d, regionState=OPENING, regionLocation=0450ab8807f5,32881,1731761492857 2024-11-16T12:51:34,828 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=4432ea1606b52a5734457a3cb3caaee3, ASSIGN because future has completed 2024-11-16T12:51:34,828 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4432ea1606b52a5734457a3cb3caaee3, server=0450ab8807f5,33961,1731761492894}] 2024-11-16T12:51:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40581 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T12:51:34,830 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=8d3e8566a59cc6153f2f93cef2a417aa, ASSIGN because future has completed 2024-11-16T12:51:34,831 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8d3e8566a59cc6153f2f93cef2a417aa, server=0450ab8807f5,32881,1731761492857}] 2024-11-16T12:51:34,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=ef3a398c9e5522ea013cb58384c3d2e5, ASSIGN because future has completed 2024-11-16T12:51:34,832 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=9c4c0c2626292bc7607edf03a64b4c11, ASSIGN because future has completed 2024-11-16T12:51:34,833 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef3a398c9e5522ea013cb58384c3d2e5, server=0450ab8807f5,33961,1731761492894}] 2024-11-16T12:51:34,833 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c4c0c2626292bc7607edf03a64b4c11, server=0450ab8807f5,39017,1731761494087}] 2024-11-16T12:51:34,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=e39334374ca56418de9e14f727ef8576, ASSIGN because future has completed 2024-11-16T12:51:34,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=75073183d2c44df3f804b5dab504292d, ASSIGN because future has completed 2024-11-16T12:51:34,838 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure e39334374ca56418de9e14f727ef8576, server=0450ab8807f5,39017,1731761494087}] 2024-11-16T12:51:34,840 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 75073183d2c44df3f804b5dab504292d, server=0450ab8807f5,32881,1731761492857}] 2024-11-16T12:51:34,982 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T12:51:34,983 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-18-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41281, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T12:51:34,988 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1731761494194.4432ea1606b52a5734457a3cb3caaee3. 2024-11-16T12:51:34,988 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 4432ea1606b52a5734457a3cb3caaee3, NAME => 'testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1731761494194.4432ea1606b52a5734457a3cb3caaee3.', STARTKEY => '3', ENDKEY => '4'} 2024-11-16T12:51:34,988 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(132): Open testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1731761494194.75073183d2c44df3f804b5dab504292d. 2024-11-16T12:51:34,989 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testLogSplittingAfterMasterRecoveryDueToZKExpiry 4432ea1606b52a5734457a3cb3caaee3 2024-11-16T12:51:34,989 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7752): Opening region: {ENCODED => 75073183d2c44df3f804b5dab504292d, NAME => 'testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1731761494194.75073183d2c44df3f804b5dab504292d.', STARTKEY => '2', ENDKEY => '3'} 2024-11-16T12:51:34,989 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1731761494194.4432ea1606b52a5734457a3cb3caaee3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:34,989 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 4432ea1606b52a5734457a3cb3caaee3 2024-11-16T12:51:34,989 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 4432ea1606b52a5734457a3cb3caaee3 2024-11-16T12:51:34,989 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testLogSplittingAfterMasterRecoveryDueToZKExpiry 75073183d2c44df3f804b5dab504292d 2024-11-16T12:51:34,989 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(898): Instantiated testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1731761494194.75073183d2c44df3f804b5dab504292d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:34,989 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7794): checking encryption for 75073183d2c44df3f804b5dab504292d 2024-11-16T12:51:34,989 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7797): checking classloading for 75073183d2c44df3f804b5dab504292d 2024-11-16T12:51:34,990 INFO [StoreOpener-4432ea1606b52a5734457a3cb3caaee3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family col of region 4432ea1606b52a5734457a3cb3caaee3 2024-11-16T12:51:34,990 INFO [StoreOpener-75073183d2c44df3f804b5dab504292d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family col of region 75073183d2c44df3f804b5dab504292d 2024-11-16T12:51:34,991 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T12:51:34,992 INFO [StoreOpener-4432ea1606b52a5734457a3cb3caaee3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4432ea1606b52a5734457a3cb3caaee3 columnFamilyName col 2024-11-16T12:51:34,992 DEBUG [StoreOpener-4432ea1606b52a5734457a3cb3caaee3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:34,992 INFO [StoreOpener-75073183d2c44df3f804b5dab504292d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75073183d2c44df3f804b5dab504292d columnFamilyName col 2024-11-16T12:51:34,992 DEBUG [StoreOpener-75073183d2c44df3f804b5dab504292d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:34,992 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-19-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47247, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T12:51:34,993 INFO [StoreOpener-75073183d2c44df3f804b5dab504292d-1 {}] regionserver.HStore(327): Store=75073183d2c44df3f804b5dab504292d/col, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:34,993 INFO [StoreOpener-4432ea1606b52a5734457a3cb3caaee3-1 {}] regionserver.HStore(327): Store=4432ea1606b52a5734457a3cb3caaee3/col, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:34,993 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 4432ea1606b52a5734457a3cb3caaee3 2024-11-16T12:51:34,993 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1038): replaying wal for 75073183d2c44df3f804b5dab504292d 2024-11-16T12:51:34,995 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/75073183d2c44df3f804b5dab504292d 2024-11-16T12:51:34,995 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/4432ea1606b52a5734457a3cb3caaee3 2024-11-16T12:51:34,995 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/4432ea1606b52a5734457a3cb3caaee3 2024-11-16T12:51:34,995 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/75073183d2c44df3f804b5dab504292d 2024-11-16T12:51:34,995 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 4432ea1606b52a5734457a3cb3caaee3 2024-11-16T12:51:34,995 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1048): stopping wal replay for 75073183d2c44df3f804b5dab504292d 2024-11-16T12:51:34,995 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 4432ea1606b52a5734457a3cb3caaee3 2024-11-16T12:51:34,995 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1060): Cleaning up temporary data for 75073183d2c44df3f804b5dab504292d 2024-11-16T12:51:34,997 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1093): writing seq id for 75073183d2c44df3f804b5dab504292d 2024-11-16T12:51:34,997 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 4432ea1606b52a5734457a3cb3caaee3 2024-11-16T12:51:35,003 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] handler.AssignRegionHandler(132): Open testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11. 2024-11-16T12:51:35,004 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(7752): Opening region: {ENCODED => 9c4c0c2626292bc7607edf03a64b4c11, NAME => 'testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11.', STARTKEY => '', ENDKEY => '1'} 2024-11-16T12:51:35,004 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/4432ea1606b52a5734457a3cb3caaee3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:51:35,004 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/75073183d2c44df3f804b5dab504292d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:51:35,004 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testLogSplittingAfterMasterRecoveryDueToZKExpiry 9c4c0c2626292bc7607edf03a64b4c11 2024-11-16T12:51:35,004 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(898): Instantiated testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:35,004 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(7794): checking encryption for 9c4c0c2626292bc7607edf03a64b4c11 2024-11-16T12:51:35,004 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(7797): checking classloading for 9c4c0c2626292bc7607edf03a64b4c11 2024-11-16T12:51:35,004 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1114): Opened 75073183d2c44df3f804b5dab504292d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67787841, jitterRate=0.0101175457239151}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T12:51:35,004 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 4432ea1606b52a5734457a3cb3caaee3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68732229, jitterRate=0.0241900235414505}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T12:51:35,004 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4432ea1606b52a5734457a3cb3caaee3 2024-11-16T12:51:35,004 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 75073183d2c44df3f804b5dab504292d 2024-11-16T12:51:35,005 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 4432ea1606b52a5734457a3cb3caaee3: Running coprocessor pre-open hook at 1731761494989Writing region info on filesystem at 1731761494989Initializing all the Stores at 1731761494990 (+1 ms)Instantiating store for column family {NAME => 'col', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761494990Cleaning up temporary data from old regions at 1731761494995 (+5 ms)Running coprocessor post-open hooks at 1731761495004 (+9 ms)Region opened successfully at 1731761495005 (+1 ms) 2024-11-16T12:51:35,005 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1006): Region open journal for 75073183d2c44df3f804b5dab504292d: Running coprocessor pre-open hook at 1731761494989Writing region info on filesystem at 1731761494989Initializing all the Stores at 1731761494990 (+1 ms)Instantiating store for column family {NAME => 'col', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761494990Cleaning up temporary data from old regions at 1731761494995 (+5 ms)Running coprocessor post-open hooks at 1731761495004 (+9 ms)Region opened successfully at 1731761495005 (+1 ms) 2024-11-16T12:51:35,006 INFO [StoreOpener-9c4c0c2626292bc7607edf03a64b4c11-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family col of region 9c4c0c2626292bc7607edf03a64b4c11 2024-11-16T12:51:35,007 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2236): Post open deploy tasks for testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1731761494194.75073183d2c44df3f804b5dab504292d., pid=16, masterSystemTime=1731761494985 2024-11-16T12:51:35,007 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1731761494194.4432ea1606b52a5734457a3cb3caaee3., pid=11, masterSystemTime=1731761494981 2024-11-16T12:51:35,007 INFO [StoreOpener-9c4c0c2626292bc7607edf03a64b4c11-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c4c0c2626292bc7607edf03a64b4c11 columnFamilyName col 2024-11-16T12:51:35,008 DEBUG [StoreOpener-9c4c0c2626292bc7607edf03a64b4c11-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:35,008 INFO [StoreOpener-9c4c0c2626292bc7607edf03a64b4c11-1 {}] regionserver.HStore(327): Store=9c4c0c2626292bc7607edf03a64b4c11/col, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:35,009 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1038): replaying wal for 9c4c0c2626292bc7607edf03a64b4c11 2024-11-16T12:51:35,009 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/9c4c0c2626292bc7607edf03a64b4c11 2024-11-16T12:51:35,010 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/9c4c0c2626292bc7607edf03a64b4c11 2024-11-16T12:51:35,010 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2266): Finished post open deploy task for testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1731761494194.75073183d2c44df3f804b5dab504292d. 2024-11-16T12:51:35,010 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(153): Opened testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1731761494194.75073183d2c44df3f804b5dab504292d. 2024-11-16T12:51:35,010 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1731761494194.8d3e8566a59cc6153f2f93cef2a417aa. 2024-11-16T12:51:35,010 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 8d3e8566a59cc6153f2f93cef2a417aa, NAME => 'testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1731761494194.8d3e8566a59cc6153f2f93cef2a417aa.', STARTKEY => '5', ENDKEY => ''} 2024-11-16T12:51:35,010 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=75073183d2c44df3f804b5dab504292d, regionState=OPEN, openSeqNum=2, regionLocation=0450ab8807f5,32881,1731761492857 2024-11-16T12:51:35,011 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testLogSplittingAfterMasterRecoveryDueToZKExpiry 8d3e8566a59cc6153f2f93cef2a417aa 2024-11-16T12:51:35,011 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1731761494194.8d3e8566a59cc6153f2f93cef2a417aa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:35,011 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 8d3e8566a59cc6153f2f93cef2a417aa 2024-11-16T12:51:35,011 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 8d3e8566a59cc6153f2f93cef2a417aa 2024-11-16T12:51:35,011 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1731761494194.4432ea1606b52a5734457a3cb3caaee3. 2024-11-16T12:51:35,011 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1731761494194.4432ea1606b52a5734457a3cb3caaee3. 2024-11-16T12:51:35,011 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1731761494194.ef3a398c9e5522ea013cb58384c3d2e5. 2024-11-16T12:51:35,012 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => ef3a398c9e5522ea013cb58384c3d2e5, NAME => 'testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1731761494194.ef3a398c9e5522ea013cb58384c3d2e5.', STARTKEY => '1', ENDKEY => '2'} 2024-11-16T12:51:35,012 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testLogSplittingAfterMasterRecoveryDueToZKExpiry ef3a398c9e5522ea013cb58384c3d2e5 2024-11-16T12:51:35,012 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1731761494194.ef3a398c9e5522ea013cb58384c3d2e5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:35,012 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=4432ea1606b52a5734457a3cb3caaee3, regionState=OPEN, openSeqNum=2, regionLocation=0450ab8807f5,33961,1731761492894 2024-11-16T12:51:35,012 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for ef3a398c9e5522ea013cb58384c3d2e5 2024-11-16T12:51:35,012 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for ef3a398c9e5522ea013cb58384c3d2e5 2024-11-16T12:51:35,014 INFO [StoreOpener-ef3a398c9e5522ea013cb58384c3d2e5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family col of region ef3a398c9e5522ea013cb58384c3d2e5 2024-11-16T12:51:35,014 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 75073183d2c44df3f804b5dab504292d, server=0450ab8807f5,32881,1731761492857 because future has completed 2024-11-16T12:51:35,015 INFO [StoreOpener-8d3e8566a59cc6153f2f93cef2a417aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family col of region 8d3e8566a59cc6153f2f93cef2a417aa 2024-11-16T12:51:35,015 INFO [StoreOpener-ef3a398c9e5522ea013cb58384c3d2e5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef3a398c9e5522ea013cb58384c3d2e5 columnFamilyName col 2024-11-16T12:51:35,015 DEBUG [StoreOpener-ef3a398c9e5522ea013cb58384c3d2e5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:35,016 INFO [StoreOpener-ef3a398c9e5522ea013cb58384c3d2e5-1 {}] regionserver.HStore(327): Store=ef3a398c9e5522ea013cb58384c3d2e5/col, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:35,016 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for ef3a398c9e5522ea013cb58384c3d2e5 2024-11-16T12:51:35,017 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/ef3a398c9e5522ea013cb58384c3d2e5 2024-11-16T12:51:35,017 INFO [StoreOpener-8d3e8566a59cc6153f2f93cef2a417aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d3e8566a59cc6153f2f93cef2a417aa columnFamilyName col 2024-11-16T12:51:35,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4432ea1606b52a5734457a3cb3caaee3, server=0450ab8807f5,33961,1731761492894 because future has completed 2024-11-16T12:51:35,017 DEBUG [StoreOpener-8d3e8566a59cc6153f2f93cef2a417aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:35,017 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/ef3a398c9e5522ea013cb58384c3d2e5 2024-11-16T12:51:35,018 INFO [StoreOpener-8d3e8566a59cc6153f2f93cef2a417aa-1 {}] regionserver.HStore(327): Store=8d3e8566a59cc6153f2f93cef2a417aa/col, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:35,018 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 8d3e8566a59cc6153f2f93cef2a417aa 2024-11-16T12:51:35,018 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for ef3a398c9e5522ea013cb58384c3d2e5 2024-11-16T12:51:35,018 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for ef3a398c9e5522ea013cb58384c3d2e5 2024-11-16T12:51:35,018 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/8d3e8566a59cc6153f2f93cef2a417aa 2024-11-16T12:51:35,019 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1048): stopping wal replay for 9c4c0c2626292bc7607edf03a64b4c11 2024-11-16T12:51:35,019 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1060): Cleaning up temporary data for 9c4c0c2626292bc7607edf03a64b4c11 2024-11-16T12:51:35,019 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/8d3e8566a59cc6153f2f93cef2a417aa 2024-11-16T12:51:35,019 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 8d3e8566a59cc6153f2f93cef2a417aa 2024-11-16T12:51:35,019 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 8d3e8566a59cc6153f2f93cef2a417aa 2024-11-16T12:51:35,020 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for ef3a398c9e5522ea013cb58384c3d2e5 2024-11-16T12:51:35,020 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1093): writing seq id for 9c4c0c2626292bc7607edf03a64b4c11 2024-11-16T12:51:35,020 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 8d3e8566a59cc6153f2f93cef2a417aa 2024-11-16T12:51:35,023 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/ef3a398c9e5522ea013cb58384c3d2e5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:51:35,024 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/8d3e8566a59cc6153f2f93cef2a417aa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:51:35,024 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/9c4c0c2626292bc7607edf03a64b4c11/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:51:35,024 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened ef3a398c9e5522ea013cb58384c3d2e5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71453397, jitterRate=0.06473858654499054}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T12:51:35,024 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ef3a398c9e5522ea013cb58384c3d2e5 2024-11-16T12:51:35,024 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for ef3a398c9e5522ea013cb58384c3d2e5: Running coprocessor pre-open hook at 1731761495012Writing region info on filesystem at 1731761495012Initializing all the Stores at 1731761495013 (+1 ms)Instantiating store for column family {NAME => 'col', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761495013Cleaning up temporary data from old regions at 1731761495018 (+5 ms)Running coprocessor post-open hooks at 1731761495024 (+6 ms)Region opened successfully at 1731761495024 2024-11-16T12:51:35,024 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 8d3e8566a59cc6153f2f93cef2a417aa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65755059, jitterRate=-0.020173266530036926}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T12:51:35,024 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8d3e8566a59cc6153f2f93cef2a417aa 2024-11-16T12:51:35,025 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 8d3e8566a59cc6153f2f93cef2a417aa: Running coprocessor pre-open hook at 1731761495011Writing region info on filesystem at 1731761495011Initializing all the Stores at 1731761495013 (+2 ms)Instantiating store for column family {NAME => 'col', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761495013Cleaning up temporary data from old regions at 1731761495019 (+6 ms)Running coprocessor post-open hooks at 1731761495025 (+6 ms)Region opened successfully at 1731761495025 2024-11-16T12:51:35,025 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1114): Opened 9c4c0c2626292bc7607edf03a64b4c11; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67183494, jitterRate=0.0011120736598968506}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T12:51:35,026 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9c4c0c2626292bc7607edf03a64b4c11 2024-11-16T12:51:35,026 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1731761494194.8d3e8566a59cc6153f2f93cef2a417aa., pid=12, masterSystemTime=1731761494985 2024-11-16T12:51:35,026 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1731761494194.ef3a398c9e5522ea013cb58384c3d2e5., pid=13, masterSystemTime=1731761494981 2024-11-16T12:51:35,026 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1006): Region open journal for 9c4c0c2626292bc7607edf03a64b4c11: Running coprocessor pre-open hook at 1731761495004Writing region info on filesystem at 1731761495004Initializing all the Stores at 1731761495005 (+1 ms)Instantiating store for column family {NAME => 'col', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761495005Cleaning up temporary data from old regions at 1731761495019 (+14 ms)Running coprocessor post-open hooks at 1731761495026 (+7 ms)Region opened successfully at 1731761495026 2024-11-16T12:51:35,027 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegionServer(2236): Post open deploy tasks for testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11., pid=14, masterSystemTime=1731761494991 2024-11-16T12:51:35,027 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=7 2024-11-16T12:51:35,027 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 75073183d2c44df3f804b5dab504292d, server=0450ab8807f5,32881,1731761492857 in 183 msec 2024-11-16T12:51:35,028 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1731761494194.8d3e8566a59cc6153f2f93cef2a417aa. 2024-11-16T12:51:35,028 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1731761494194.8d3e8566a59cc6153f2f93cef2a417aa. 2024-11-16T12:51:35,030 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1731761494194.ef3a398c9e5522ea013cb58384c3d2e5. 2024-11-16T12:51:35,030 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1731761494194.ef3a398c9e5522ea013cb58384c3d2e5. 2024-11-16T12:51:35,030 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=ef3a398c9e5522ea013cb58384c3d2e5, regionState=OPEN, openSeqNum=2, regionLocation=0450ab8807f5,33961,1731761492894 2024-11-16T12:51:35,030 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=8d3e8566a59cc6153f2f93cef2a417aa, regionState=OPEN, openSeqNum=2, regionLocation=0450ab8807f5,32881,1731761492857 2024-11-16T12:51:35,032 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegionServer(2266): Finished post open deploy task for testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11. 2024-11-16T12:51:35,032 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] handler.AssignRegionHandler(153): Opened testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11. 2024-11-16T12:51:35,032 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1731761494194.e39334374ca56418de9e14f727ef8576. 2024-11-16T12:51:35,032 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => e39334374ca56418de9e14f727ef8576, NAME => 'testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1731761494194.e39334374ca56418de9e14f727ef8576.', STARTKEY => '4', ENDKEY => '5'} 2024-11-16T12:51:35,033 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=75073183d2c44df3f804b5dab504292d, ASSIGN in 357 msec 2024-11-16T12:51:35,033 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testLogSplittingAfterMasterRecoveryDueToZKExpiry e39334374ca56418de9e14f727ef8576 2024-11-16T12:51:35,033 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1731761494194.e39334374ca56418de9e14f727ef8576.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:35,033 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for e39334374ca56418de9e14f727ef8576 2024-11-16T12:51:35,033 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for e39334374ca56418de9e14f727ef8576 2024-11-16T12:51:35,034 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9c4c0c2626292bc7607edf03a64b4c11, regionState=OPEN, openSeqNum=2, regionLocation=0450ab8807f5,39017,1731761494087 2024-11-16T12:51:35,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8d3e8566a59cc6153f2f93cef2a417aa, server=0450ab8807f5,32881,1731761492857 because future has completed 2024-11-16T12:51:35,035 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef3a398c9e5522ea013cb58384c3d2e5, server=0450ab8807f5,33961,1731761492894 because future has completed 2024-11-16T12:51:35,036 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=8 2024-11-16T12:51:35,036 INFO [StoreOpener-e39334374ca56418de9e14f727ef8576-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family col of region e39334374ca56418de9e14f727ef8576 2024-11-16T12:51:35,036 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c4c0c2626292bc7607edf03a64b4c11, server=0450ab8807f5,39017,1731761494087 because future has completed 2024-11-16T12:51:35,037 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 4432ea1606b52a5734457a3cb3caaee3, server=0450ab8807f5,33961,1731761492894 in 200 msec 2024-11-16T12:51:35,038 INFO [StoreOpener-e39334374ca56418de9e14f727ef8576-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e39334374ca56418de9e14f727ef8576 columnFamilyName col 2024-11-16T12:51:35,038 DEBUG [StoreOpener-e39334374ca56418de9e14f727ef8576-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:35,039 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-16T12:51:35,039 INFO [StoreOpener-e39334374ca56418de9e14f727ef8576-1 {}] regionserver.HStore(327): Store=e39334374ca56418de9e14f727ef8576/col, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:35,039 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for e39334374ca56418de9e14f727ef8576 2024-11-16T12:51:35,040 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=6 2024-11-16T12:51:35,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=4432ea1606b52a5734457a3cb3caaee3, ASSIGN in 367 msec 2024-11-16T12:51:35,040 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure ef3a398c9e5522ea013cb58384c3d2e5, server=0450ab8807f5,33961,1731761492894 in 204 msec 2024-11-16T12:51:35,040 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/e39334374ca56418de9e14f727ef8576 2024-11-16T12:51:35,041 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/e39334374ca56418de9e14f727ef8576 2024-11-16T12:51:35,041 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for e39334374ca56418de9e14f727ef8576 2024-11-16T12:51:35,041 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for e39334374ca56418de9e14f727ef8576 2024-11-16T12:51:35,041 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 8d3e8566a59cc6153f2f93cef2a417aa, server=0450ab8807f5,32881,1731761492857 in 204 msec 2024-11-16T12:51:35,042 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=5 2024-11-16T12:51:35,042 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 9c4c0c2626292bc7607edf03a64b4c11, server=0450ab8807f5,39017,1731761494087 in 204 msec 2024-11-16T12:51:35,042 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=8d3e8566a59cc6153f2f93cef2a417aa, ASSIGN in 370 msec 2024-11-16T12:51:35,043 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for e39334374ca56418de9e14f727ef8576 2024-11-16T12:51:35,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=ef3a398c9e5522ea013cb58384c3d2e5, ASSIGN in 371 msec 2024-11-16T12:51:35,044 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=9c4c0c2626292bc7607edf03a64b4c11, ASSIGN in 372 msec 2024-11-16T12:51:35,045 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/e39334374ca56418de9e14f727ef8576/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:51:35,046 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened e39334374ca56418de9e14f727ef8576; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74701050, jitterRate=0.11313238739967346}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T12:51:35,046 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e39334374ca56418de9e14f727ef8576 2024-11-16T12:51:35,046 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for e39334374ca56418de9e14f727ef8576: Running coprocessor pre-open hook at 1731761495033Writing region info on filesystem at 1731761495033Initializing all the Stores at 1731761495035 (+2 ms)Instantiating store for column family {NAME => 'col', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761495035Cleaning up temporary data from old regions at 1731761495041 (+6 ms)Running coprocessor post-open hooks at 1731761495046 (+5 ms)Region opened successfully at 1731761495046 2024-11-16T12:51:35,047 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1731761494194.e39334374ca56418de9e14f727ef8576., pid=15, masterSystemTime=1731761494991 2024-11-16T12:51:35,049 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1731761494194.e39334374ca56418de9e14f727ef8576. 2024-11-16T12:51:35,049 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1731761494194.e39334374ca56418de9e14f727ef8576. 2024-11-16T12:51:35,050 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=e39334374ca56418de9e14f727ef8576, regionState=OPEN, openSeqNum=2, regionLocation=0450ab8807f5,39017,1731761494087 2024-11-16T12:51:35,052 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure e39334374ca56418de9e14f727ef8576, server=0450ab8807f5,39017,1731761494087 because future has completed 2024-11-16T12:51:35,056 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=9 2024-11-16T12:51:35,056 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure e39334374ca56418de9e14f727ef8576, server=0450ab8807f5,39017,1731761494087 in 216 msec 2024-11-16T12:51:35,058 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=4 2024-11-16T12:51:35,058 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=e39334374ca56418de9e14f727ef8576, ASSIGN in 386 msec 2024-11-16T12:51:35,059 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T12:51:35,059 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testLogSplittingAfterMasterRecoveryDueToZKExpiry","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731761495059"}]},"ts":"1731761495059"} 2024-11-16T12:51:35,061 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testLogSplittingAfterMasterRecoveryDueToZKExpiry, state=ENABLED in hbase:meta 2024-11-16T12:51:35,063 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T12:51:35,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry in 868 msec 2024-11-16T12:51:35,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40581 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T12:51:35,340 INFO [RPCClient-NioEventLoopGroup-6-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testLogSplittingAfterMasterRecoveryDueToZKExpiry completed 2024-11-16T12:51:35,342 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T12:51:35,351 DEBUG [RPCClient-NioEventLoopGroup-6-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testLogSplittingAfterMasterRecoveryDueToZKExpiry', row='\x00\x00\x00\x00', locateType=CURRENT is [region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11., hostname=0450ab8807f5,39017,1731761494087, seqNum=2] 2024-11-16T12:51:35,352 DEBUG [RPCClient-NioEventLoopGroup-6-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T12:51:35,355 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-19-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41488, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T12:51:35,369 ERROR [Time-limited test {}] master.HMaster(3218): Master server abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-16T12:51:35,369 ERROR [Time-limited test {}] master.HMaster(3223): ***** ABORTING master 0450ab8807f5,40581,1731761492744: Test recovery from zk session expired ***** org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired at org.apache.hadoop.hbase.TestZooKeeper.testLogSplittingAfterMasterRecoveryDueToZKExpiry(TestZooKeeper.java:269) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:35,369 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master '0450ab8807f5,40581,1731761492744' ***** 2024-11-16T12:51:35,369 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: Stopped by Time-limited test 2024-11-16T12:51:35,369 INFO [M:0;0450ab8807f5:40581 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T12:51:35,369 INFO [M:0;0450ab8807f5:40581 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;0450ab8807f5:40581. 2024-11-16T12:51:35,370 DEBUG [M:0;0450ab8807f5:40581 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:51:35,370 DEBUG [M:0;0450ab8807f5:40581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:35,370 INFO [M:0;0450ab8807f5:40581 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T12:51:35,370 DEBUG [M:0;0450ab8807f5:40581 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T12:51:35,370 DEBUG [M:0;0450ab8807f5:40581 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T12:51:35,370 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T12:51:35,370 DEBUG [master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.large.0-1731761493282 {}] cleaner.HFileCleaner(306): Exit Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.large.0-1731761493282,5,FailOnTimeoutGroup] 2024-11-16T12:51:35,370 DEBUG [master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.small.0-1731761493282 {}] cleaner.HFileCleaner(306): Exit Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.small.0-1731761493282,5,FailOnTimeoutGroup] 2024-11-16T12:51:35,370 INFO [M:0;0450ab8807f5:40581 {}] hbase.ChoreService(370): Chore service for: master/0450ab8807f5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T12:51:35,370 INFO [M:0;0450ab8807f5:40581 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T12:51:35,370 DEBUG [M:0;0450ab8807f5:40581 {}] master.HMaster(1795): Stopping service threads 2024-11-16T12:51:35,370 INFO [M:0;0450ab8807f5:40581 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T12:51:35,370 INFO [M:0;0450ab8807f5:40581 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T12:51:35,371 INFO [M:0;0450ab8807f5:40581 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=true 2024-11-16T12:51:35,371 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T12:51:35,381 DEBUG [HMaster-EventLoopGroup-15-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0450ab8807f5,40581,1731761492744 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:35,399 DEBUG [HMaster-EventLoopGroup-15-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0450ab8807f5,40581,1731761492744 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:35,432 DEBUG [HMaster-EventLoopGroup-15-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0450ab8807f5,40581,1731761492744 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:35,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40581-0x10144fc6ca60012, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:35,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39017-0x10144fc6ca60017, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:35,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:35,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39017-0x10144fc6ca60017, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:35,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40581-0x10144fc6ca60012, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:35,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:35,436 DEBUG [M:0;0450ab8807f5:40581 {}] zookeeper.ZKUtil(347): master:40581-0x10144fc6ca60012, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-16T12:51:35,436 WARN [M:0;0450ab8807f5:40581 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-16T12:51:35,436 DEBUG [RS:0;0450ab8807f5:32881 {}] regionserver.HRegionServer(2589): No master found; retry 2024-11-16T12:51:35,436 DEBUG [pool-593-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10144fc6ca60015, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:35,437 DEBUG [pool-593-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10144fc6ca60015, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:35,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32881-0x10144fc6ca60014, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:35,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32881-0x10144fc6ca60014, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:35,437 DEBUG [RS:1;0450ab8807f5:33961 {}] regionserver.HRegionServer(2589): No master found; retry 2024-11-16T12:51:35,437 DEBUG [RS:2;0450ab8807f5:39017 {}] regionserver.HRegionServer(2589): No master found; retry 2024-11-16T12:51:35,437 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:32881-0x10144fc6ca60014, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:35,437 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39017-0x10144fc6ca60017, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:35,437 INFO [M:0;0450ab8807f5:40581 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/.lastflushedseqids 2024-11-16T12:51:35,437 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33961-0x10144fc6ca60015, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:35,438 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:35,438 DEBUG [zk-event-processor-pool-0 {}] master.ActiveMasterManager(204): No master available. Notifying waiting threads 2024-11-16T12:51:35,440 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testLogSplittingAfterMasterRecoveryDueToZKExpiry,, stopping at row=testLogSplittingAfterMasterRecoveryDueToZKExpiry ,, for max=2147483647 with caching=100 2024-11-16T12:51:35,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741901_1079 (size=472) 2024-11-16T12:51:35,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741901_1079 (size=472) 2024-11-16T12:51:35,444 INFO [M:0;0450ab8807f5:40581 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T12:51:35,444 INFO [M:0;0450ab8807f5:40581 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=true 2024-11-16T12:51:35,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39017-0x10144fc6ca60017, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:35,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:35,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:35,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40581-0x10144fc6ca60012, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:35,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39017-0x10144fc6ca60017, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:35,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32881-0x10144fc6ca60014, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:35,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32881-0x10144fc6ca60014, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:35,445 DEBUG [pool-593-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10144fc6ca60015, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:35,445 DEBUG [pool-593-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10144fc6ca60015, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:35,446 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T12:51:35,446 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T12:51:35,447 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0450ab8807f5,44015,1731761492818 from backup master directory 2024-11-16T12:51:35,448 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 6 regions for table testLogSplittingAfterMasterRecoveryDueToZKExpiry 2024-11-16T12:51:35,448 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11. 2024-11-16T12:51:35,449 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 0450ab8807f5,39017,1731761494087 2024-11-16T12:51:35,449 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 0450ab8807f5,39017,1731761494087: Simulated kill ***** 2024-11-16T12:51:35,449 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-16T12:51:35,449 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-16T12:51:35,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741887_1065 (size=72133) 2024-11-16T12:51:35,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741887_1065 (size=72133) 2024-11-16T12:51:35,450 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-16T12:51:35,450 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-16T12:51:35,450 DEBUG [M:0;0450ab8807f5:40581 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T12:51:35,450 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-16T12:51:35,450 INFO [M:0;0450ab8807f5:40581 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:35,450 DEBUG [M:0;0450ab8807f5:40581 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:35,450 DEBUG [M:0;0450ab8807f5:40581 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T12:51:35,450 DEBUG [M:0;0450ab8807f5:40581 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:35,451 ERROR [M:0;0450ab8807f5:40581 {}] regionserver.HRegion(1960): Memstore data size is 63644 in region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:35,451 INFO [M:0;0450ab8807f5:40581 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:35,451 DEBUG [M:0;0450ab8807f5:40581 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731761495450Disabling compacts and flushes for region at 1731761495450Disabling writes for close at 1731761495450Writing region close event to WAL at 1731761495451 (+1 ms)Closed at 1731761495451 2024-11-16T12:51:35,451 INFO [M:0;0450ab8807f5:40581 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-16T12:51:35,451 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T12:51:35,451 INFO [M:0;0450ab8807f5:40581 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40581 2024-11-16T12:51:35,451 INFO [M:0;0450ab8807f5:40581 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T12:51:35,452 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 415056816 }, "NonHeapMemoryUsage": { "committed": 165543936, "init": 7667712, "max": -1, "used": 162950848 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "0450ab8807f5", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 13406, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 47, "ProcessCallTime_min": 0, "ProcessCallTime_max": 5, "ProcessCallTime_mean": 1, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 1, "ProcessCallTime_75th_percentile": 1, "ProcessCallTime_90th_percentile": 3, "ProcessCallTime_95th_percentile": 4, "ProcessCallTime_98th_percentile": 5, "ProcessCallTime_99th_percentile": 5, "ProcessCallTime_99.9th_percentile": 5, "ProcessCallTime_TimeRangeCount_0-1": 17, "ProcessCallTime_TimeRangeCount_1-3": 2, "ProcessCallTime_TimeRangeCount_3-10": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 47, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 1, "QueueCallTime_95th_percentile": 1, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 20, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 47, "TotalCallTime_min": 1, "TotalCallTime_max": 5, "TotalCallTime_mean": 1, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 1, "TotalCallTime_75th_percentile": 1, "TotalCallTime_90th_percentile": 3, "TotalCallTime_95th_percentile": 4, "TotalCallTime_98th_percentile": 5, "TotalCallTime_99th_percentile": 5, "TotalCallTime_99.9th_percentile": 5, "TotalCallTime_TimeRangeCount_0-1": 16, "TotalCallTime_TimeRangeCount_1-3": 3, "TotalCallTime_TimeRangeCount_3-10": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 47, "ResponseSize_min": 0, "ResponseSize_max": 1098, "ResponseSize_mean": 86, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 3, "ResponseSize_75th_percentile": 23, "ResponseSize_90th_percentile": 175, "ResponseSize_95th_percentile": 176, "ResponseSize_98th_percentile": 760, "ResponseSize_99th_percentile": 929, "ResponseSize_99.9th_percentile": 1081, "ResponseSize_SizeRangeCount_0-10": 12, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 4, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 47, "RequestSize_min": 89, "RequestSize_max": 382, "RequestSize_mean": 151, "RequestSize_25th_percentile": 100, "RequestSize_median": 121, "RequestSize_75th_percentile": 131, "RequestSize_90th_percentile": 274, "RequestSize_95th_percentile": 323, "RequestSize_98th_percentile": 366, "RequestSize_99th_percentile": 374, "RequestSize_99.9th_percentile": 381, "RequestSize_SizeRangeCount_10-100": 5, "RequestSize_SizeRangeCount_100-1000": 15, "sentBytes": 3296 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "0450ab8807f5", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:53208", "tag.serverName": "0450ab8807f5,36743,1731761465323", "tag.clusterId": "be09e536-22d9-488d-a733-7968a5bb48e1", "tag.Context": "regionserver", "tag.Hostname": "0450ab8807f5", "regionCount": 0, "storeCount": 0, "hlogFileCount": 7, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731761465323, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 55730, "localBytesRead": 55730, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 32470, "ByteBuffAllocatorPoolAllocationBytes": 66560, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 7, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 1, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 6, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 1, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 6, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 5153, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 74, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 1, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-16T12:51:35,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40581-0x10144fc6ca60012, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:35,453 DEBUG [pool-593-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10144fc6ca60015, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:35,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0450ab8807f5,44015,1731761492818 2024-11-16T12:51:35,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39017-0x10144fc6ca60017, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:35,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:35,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32881-0x10144fc6ca60014, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:35,453 WARN [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T12:51:35,453 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0450ab8807f5,44015,1731761492818 2024-11-16T12:51:35,453 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0450ab8807f5,39017,1731761494087' ***** 2024-11-16T12:51:35,453 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-16T12:51:35,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-19-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0450ab8807f5,39017,1731761494087 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:35,456 DEBUG [RPCClient-NioEventLoopGroup-6-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11., hostname=0450ab8807f5,39017,1731761494087, seqNum=2 , the old value is region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11., hostname=0450ab8807f5,39017,1731761494087, seqNum=2, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0450ab8807f5,39017,1731761494087 aborting 2024-11-16T12:51:35,457 DEBUG [RPCClient-NioEventLoopGroup-6-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11., hostname=0450ab8807f5,39017,1731761494087, seqNum=2 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0450ab8807f5,39017,1731761494087 aborting 2024-11-16T12:51:35,457 DEBUG [RPCClient-NioEventLoopGroup-6-12 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11., hostname=0450ab8807f5,39017,1731761494087, seqNum=2 from cache 2024-11-16T12:51:35,463 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T12:51:35,465 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:35,465 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T12:51:35,474 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=2) cost 9ms. 2024-11-16T12:51:35,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741902_1080 (size=196) 2024-11-16T12:51:35,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741902_1080 (size=196) 2024-11-16T12:51:35,493 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T12:51:35,493 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T12:51:35,494 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:51:35,500 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(342): old store file tracker DEFAULT is the same with new store file tracker, skip migration 2024-11-16T12:51:35,503 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(316): Renamed hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,40581,1731761492744 to hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,40581,1731761492744-dead as it is dead 2024-11-16T12:51:35,503 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,40581,1731761492744-dead/0450ab8807f5%2C40581%2C1731761492744.1731761493040 2024-11-16T12:51:35,504 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,40581,1731761492744-dead/0450ab8807f5%2C40581%2C1731761492744.1731761493040 after 1ms 2024-11-16T12:51:35,504 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(328): Renamed hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,40581,1731761492744-dead/0450ab8807f5%2C40581%2C1731761492744.1731761493040 to hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.wals/0450ab8807f5%2C40581%2C1731761492744.1731761493040 2024-11-16T12:51:35,504 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(330): Delete empty local region wal dir hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,40581,1731761492744-dead 2024-11-16T12:51:35,505 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,44015,1731761492818 2024-11-16T12:51:35,505 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T12:51:35,507 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0450ab8807f5%2C44015%2C1731761492818, suffix=, logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,44015,1731761492818, archiveDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/oldWALs, maxLogs=10 2024-11-16T12:51:35,519 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,44015,1731761492818/0450ab8807f5%2C44015%2C1731761492818.1731761495507, exclude list is [], retry=0 2024-11-16T12:51:35,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-19-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43257,DS-06d91055-02a4-40d7-bddb-15aee81f02ac,DISK] 2024-11-16T12:51:35,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-19-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36961,DS-7ef9e9a6-250e-467c-989b-a9ffcab2299e,DISK] 2024-11-16T12:51:35,524 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,44015,1731761492818/0450ab8807f5%2C44015%2C1731761492818.1731761495507 2024-11-16T12:51:35,524 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:46667:46667)] 2024-11-16T12:51:35,524 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-16T12:51:35,524 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:35,524 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:35,524 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:35,526 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:35,526 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-16T12:51:35,526 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:35,527 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:35,527 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:35,528 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-16T12:51:35,528 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:35,528 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:35,529 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:35,529 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-16T12:51:35,529 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:35,530 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:35,530 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:35,531 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-16T12:51:35,531 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:35,531 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:35,531 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:35,532 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(5516): Found 1 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.wals 2024-11-16T12:51:35,532 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.wals/0450ab8807f5%2C40581%2C1731761492744.1731761493040 2024-11-16T12:51:35,542 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(5793): Applied 163, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=131, path=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.wals/0450ab8807f5%2C40581%2C1731761492744.1731761493040 2024-11-16T12:51:35,542 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=62.15 KB heapSize=75.20 KB 2024-11-16T12:51:35,557 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8212139fc0b7435da7ceac26d1f3ea9f is 82, key is hbase:meta,,1/info:regioninfo/1731761493949/Put/seqid=0 2024-11-16T12:51:35,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40581-0x10144fc6ca60012, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:35,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40581-0x10144fc6ca60012, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:35,562 INFO [M:0;0450ab8807f5:40581 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T12:51:35,562 DEBUG [RPCClient-NioEventLoopGroup-6-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testLogSplittingAfterMasterRecoveryDueToZKExpiry', row='', locateType=CURRENT is [region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11., hostname=0450ab8807f5,39017,1731761494087, seqNum=2] 2024-11-16T12:51:35,563 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-19-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0450ab8807f5,39017,1731761494087 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:35,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741904_1082 (size=5672) 2024-11-16T12:51:35,564 DEBUG [RPCClient-NioEventLoopGroup-6-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11., hostname=0450ab8807f5,39017,1731761494087, seqNum=2 , the old value is region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11., hostname=0450ab8807f5,39017,1731761494087, seqNum=2, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0450ab8807f5,39017,1731761494087 aborting 2024-11-16T12:51:35,564 DEBUG [RPCClient-NioEventLoopGroup-6-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11., hostname=0450ab8807f5,39017,1731761494087, seqNum=2 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0450ab8807f5,39017,1731761494087 aborting 2024-11-16T12:51:35,564 DEBUG [RPCClient-NioEventLoopGroup-6-12 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11., hostname=0450ab8807f5,39017,1731761494087, seqNum=2 from cache 2024-11-16T12:51:35,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741904_1082 (size=5672) 2024-11-16T12:51:35,564 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8212139fc0b7435da7ceac26d1f3ea9f 2024-11-16T12:51:35,592 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5743e86ccc1c4d0ca911d5d4fe968938 is 1199, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731761495063/Put/seqid=0 2024-11-16T12:51:35,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741905_1083 (size=68119) 2024-11-16T12:51:35,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741905_1083 (size=68119) 2024-11-16T12:51:35,599 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=61.47 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5743e86ccc1c4d0ca911d5d4fe968938 2024-11-16T12:51:35,626 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/eb12e0e667ff4639a052f2e1eac864e8 is 69, key is 0450ab8807f5,32881,1731761492857/rs:state/1731761493369/Put/seqid=0 2024-11-16T12:51:35,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741906_1084 (size=5294) 2024-11-16T12:51:35,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741906_1084 (size=5294) 2024-11-16T12:51:35,637 INFO [RS:2;0450ab8807f5:39017 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T12:51:35,637 INFO [RS:2;0450ab8807f5:39017 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-16T12:51:35,637 INFO [RS:2;0450ab8807f5:39017 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-16T12:51:35,637 INFO [RS:2;0450ab8807f5:39017 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T12:51:35,637 INFO [RS:2;0450ab8807f5:39017 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;0450ab8807f5:39017. 2024-11-16T12:51:35,638 DEBUG [RS:2;0450ab8807f5:39017 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:51:35,638 DEBUG [RS:2;0450ab8807f5:39017 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:35,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741900_1078 (size=1682) 2024-11-16T12:51:35,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741900_1078 (size=1682) 2024-11-16T12:51:35,641 DEBUG [RS:2;0450ab8807f5:39017 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:35,641 INFO [RS:2;0450ab8807f5:39017 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:51:35,641 INFO [RS:2;0450ab8807f5:39017 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39017 2024-11-16T12:51:35,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T12:51:35,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39017-0x10144fc6ca60017, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0450ab8807f5,39017,1731761494087 2024-11-16T12:51:35,653 INFO [RS:2;0450ab8807f5:39017 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T12:51:35,664 INFO [regionserver/0450ab8807f5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:51:35,738 DEBUG [RS:0;0450ab8807f5:32881 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T12:51:35,738 DEBUG [RS:1;0450ab8807f5:33961 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T12:51:35,740 INFO [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42297, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T12:51:35,740 INFO [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45377, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T12:51:35,740 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44015 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:35,741 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44015 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:35,762 INFO [RS:2;0450ab8807f5:39017 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T12:51:35,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39017-0x10144fc6ca60017, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:35,762 INFO [RS:2;0450ab8807f5:39017 {}] regionserver.HRegionServer(1031): Exiting; stopping=0450ab8807f5,39017,1731761494087; zookeeper connection closed. 2024-11-16T12:51:35,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39017-0x10144fc6ca60017, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:35,762 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@652fe2c5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@652fe2c5 2024-11-16T12:51:35,772 DEBUG [RPCClient-NioEventLoopGroup-6-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testLogSplittingAfterMasterRecoveryDueToZKExpiry', row='', locateType=CURRENT is [region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11., hostname=0450ab8807f5,39017,1731761494087, seqNum=2] 2024-11-16T12:51:35,773 WARN [RPCClient-NioEventLoopGroup-6-12 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 0450ab8807f5:39017 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 0450ab8807f5/172.17.0.2:39017 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:35,774 DEBUG [RPCClient-NioEventLoopGroup-6-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11., hostname=0450ab8807f5,39017,1731761494087, seqNum=2 , the old value is region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11., hostname=0450ab8807f5,39017,1731761494087, seqNum=2, error=java.net.ConnectException: Call to address=0450ab8807f5:39017 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 0450ab8807f5/172.17.0.2:39017 2024-11-16T12:51:35,774 DEBUG [RPCClient-NioEventLoopGroup-6-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11., hostname=0450ab8807f5,39017,1731761494087, seqNum=2 is java.net.ConnectException: Connection refused 2024-11-16T12:51:35,774 DEBUG [RPCClient-NioEventLoopGroup-6-12 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11., hostname=0450ab8807f5,39017,1731761494087, seqNum=2 from cache 2024-11-16T12:51:35,774 DEBUG [RPCClient-NioEventLoopGroup-6-12 {}] ipc.FailedServers(52): Added failed server with address 0450ab8807f5:39017 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 0450ab8807f5/172.17.0.2:39017 2024-11-16T12:51:35,843 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44015 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:35,843 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44015 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:35,946 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44015 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:35,946 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44015 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:36,032 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=195 B at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/eb12e0e667ff4639a052f2e1eac864e8 2024-11-16T12:51:36,041 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8212139fc0b7435da7ceac26d1f3ea9f as hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8212139fc0b7435da7ceac26d1f3ea9f 2024-11-16T12:51:36,049 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44015 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:36,049 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8212139fc0b7435da7ceac26d1f3ea9f, entries=8, sequenceid=131, filesize=5.5 K 2024-11-16T12:51:36,049 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44015 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:36,050 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5743e86ccc1c4d0ca911d5d4fe968938 as hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5743e86ccc1c4d0ca911d5d4fe968938 2024-11-16T12:51:36,057 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5743e86ccc1c4d0ca911d5d4fe968938, entries=152, sequenceid=131, filesize=66.5 K 2024-11-16T12:51:36,058 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/eb12e0e667ff4639a052f2e1eac864e8 as hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/eb12e0e667ff4639a052f2e1eac864e8 2024-11-16T12:51:36,066 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/eb12e0e667ff4639a052f2e1eac864e8, entries=3, sequenceid=131, filesize=5.2 K 2024-11-16T12:51:36,066 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(3140): Finished flush of dataSize ~62.15 KB/63644, heapSize ~74.90 KB/76696, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 524ms, sequenceid=131, compaction requested=false; wal=null 2024-11-16T12:51:36,066 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.wals/0450ab8807f5%2C40581%2C1731761492744.1731761493040 2024-11-16T12:51:36,068 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:36,068 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:36,068 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T12:51:36,070 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:36,072 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/131.seqid, newMaxSeqId=131, maxSeqId=1 2024-11-16T12:51:36,073 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=132; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74918683, jitterRate=0.11637537181377411}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T12:51:36,074 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731761495524Initializing all the Stores at 1731761495525 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761495525Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761495526 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761495526Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761495526Obtaining lock to block concurrent updates at 1731761495542 (+16 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731761495542Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=63644, getHeapSize=76936, getOffHeapSize=0, getCellsCount=163 at 1731761495542Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731761495542Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731761495542Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731761495556 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731761495556Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731761495571 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731761495591 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731761495591Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731761495607 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731761495625 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731761495626 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58791e35: reopening flushed file at 1731761496040 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37e75c6f: reopening flushed file at 1731761496049 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3acd89e4: reopening flushed file at 1731761496057 (+8 ms)Finished flush of dataSize ~62.15 KB/63644, heapSize ~74.90 KB/76696, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 524ms, sequenceid=131, compaction requested=false; wal=null at 1731761496066 (+9 ms)Cleaning up temporary data from old regions at 1731761496068 (+2 ms)Region opened successfully at 1731761496074 (+6 ms) 2024-11-16T12:51:36,075 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-16T12:51:36,078 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e38a97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0450ab8807f5/172.17.0.2:0 2024-11-16T12:51:36,082 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(884): The info family in master local region already has data in it, skip migrating... 2024-11-16T12:51:36,082 DEBUG [RPCClient-NioEventLoopGroup-6-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testLogSplittingAfterMasterRecoveryDueToZKExpiry', row='', locateType=CURRENT is [region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11., hostname=0450ab8807f5,39017,1731761494087, seqNum=2] 2024-11-16T12:51:36,082 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-16T12:51:36,083 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-16T12:51:36,083 DEBUG [RPCClient-NioEventLoopGroup-6-10 {}] ipc.AbstractRpcClient(357): Not trying to connect to 0450ab8807f5:39017 this server is in the failed servers list 2024-11-16T12:51:36,083 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T12:51:36,083 DEBUG [RPCClient-NioEventLoopGroup-6-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11., hostname=0450ab8807f5,39017,1731761494087, seqNum=2 , the old value is region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11., hostname=0450ab8807f5,39017,1731761494087, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=0450ab8807f5:39017 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 0450ab8807f5:39017 2024-11-16T12:51:36,083 DEBUG [RPCClient-NioEventLoopGroup-6-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11., hostname=0450ab8807f5,39017,1731761494087, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 0450ab8807f5:39017 2024-11-16T12:51:36,083 DEBUG [RPCClient-NioEventLoopGroup-6-10 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11., hostname=0450ab8807f5,39017,1731761494087, seqNum=2 from cache 2024-11-16T12:51:36,083 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T12:51:36,089 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T12:51:36,090 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry 2024-11-16T12:51:36,090 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-16T12:51:36,090 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-16T12:51:36,093 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=0450ab8807f5,32881,1731761492857, regionLocation=0450ab8807f5,32881,1731761492857, openSeqNum=2 2024-11-16T12:51:36,093 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=0450ab8807f5,32881,1731761492857, table=hbase:meta, region=1588230740 2024-11-16T12:51:36,094 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0450ab8807f5,32881,1731761492857, state=OPEN 2024-11-16T12:51:36,149 DEBUG [pool-593-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10144fc6ca60015, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:36,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32881-0x10144fc6ca60014, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:36,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:36,149 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:36,149 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:36,149 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:36,151 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44015 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:36,152 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44015 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T12:51:36,153 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 3 possibly 'live' servers, and 0 'splitting'. 2024-11-16T12:51:36,153 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ServerManager(517): Registering regionserver=0450ab8807f5,33961,1731761492894 2024-11-16T12:51:36,154 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ServerManager(517): Registering regionserver=0450ab8807f5,32881,1731761492857 2024-11-16T12:51:36,162 INFO [regionserver/0450ab8807f5:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-16T12:51:36,162 INFO [regionserver/0450ab8807f5:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-16T12:51:36,169 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0450ab8807f5,39017,1731761494087 already deleted, retry=false 2024-11-16T12:51:36,170 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ServerManager(695): Processing expiration of 0450ab8807f5,39017,1731761494087 on 0450ab8807f5,44015,1731761492818 2024-11-16T12:51:36,171 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 0450ab8807f5,39017,1731761494087, splitWal=true, meta=false 2024-11-16T12:51:36,171 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=17 for 0450ab8807f5,39017,1731761494087 (carryingMeta=false) 0450ab8807f5,39017,1731761494087/CRASHED/regionCount=0/lock=java.util.concurrent.locks.ReentrantReadWriteLock@6122f9b1[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-16T12:51:36,173 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-16T12:51:36,178 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-16T12:51:36,178 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-16T12:51:36,179 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-16T12:51:36,186 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-16T12:51:36,186 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-16T12:51:36,187 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-16T12:51:36,194 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-16T12:51:36,195 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-16T12:51:36,203 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-16T12:51:36,205 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-16T12:51:36,211 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-16T12:51:36,211 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0450ab8807f5,44015,1731761492818, sessionid=0x10144fc6ca60013, setting cluster-up flag (Was=true) 2024-11-16T12:51:36,220 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-16T12:51:36,222 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0450ab8807f5,44015,1731761492818 2024-11-16T12:51:36,228 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-16T12:51:36,229 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0450ab8807f5,44015,1731761492818 2024-11-16T12:51:36,231 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ServerManager(1190): begin to load .lastflushedseqids at hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/.lastflushedseqids 2024-11-16T12:51:36,233 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-16T12:51:36,233 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0450ab8807f5,44015,1731761492818 Number of backup masters: 0 Number of live region servers: 2 0450ab8807f5,32881,1731761492857 0450ab8807f5,33961,1731761492894 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-16T12:51:36,234 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:36,234 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:36,234 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:36,234 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:36,235 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0450ab8807f5:0, corePoolSize=10, maxPoolSize=10 2024-11-16T12:51:36,235 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:36,235 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0450ab8807f5:0, corePoolSize=2, maxPoolSize=2 2024-11-16T12:51:36,235 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:36,235 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731761526235 2024-11-16T12:51:36,235 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-16T12:51:36,235 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-16T12:51:36,235 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-16T12:51:36,235 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-16T12:51:36,235 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-16T12:51:36,235 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-16T12:51:36,236 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:36,236 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-16T12:51:36,236 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-16T12:51:36,236 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-16T12:51:36,236 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=17, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 0450ab8807f5,39017,1731761494087, splitWal=true, meta=false 2024-11-16T12:51:36,236 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-16T12:51:36,236 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-16T12:51:36,236 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.large.0-1731761496236,5,FailOnTimeoutGroup] 2024-11-16T12:51:36,236 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.small.0-1731761496236,5,FailOnTimeoutGroup] 2024-11-16T12:51:36,236 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:36,236 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-16T12:51:36,237 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:36,237 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:36,237 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731761496237, completionTime=-1 2024-11-16T12:51:36,237 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-16T12:51:36,237 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T12:51:36,237 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T12:51:36,237 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,32881,1731761492857, seqNum=-1] 2024-11-16T12:51:36,238 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T12:51:36,239 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56451, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T12:51:36,242 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c4c0c2626292bc7607edf03a64b4c11, regionState=OPEN, lastHost=0450ab8807f5,39017,1731761494087, regionLocation=0450ab8807f5,39017,1731761494087, openSeqNum=2 2024-11-16T12:51:36,242 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef3a398c9e5522ea013cb58384c3d2e5, regionState=OPEN, lastHost=0450ab8807f5,33961,1731761492894, regionLocation=0450ab8807f5,33961,1731761492894, openSeqNum=2 2024-11-16T12:51:36,242 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.RegionStateStore(171): Load hbase:meta entry region=75073183d2c44df3f804b5dab504292d, regionState=OPEN, lastHost=0450ab8807f5,32881,1731761492857, regionLocation=0450ab8807f5,32881,1731761492857, openSeqNum=2 2024-11-16T12:51:36,243 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4432ea1606b52a5734457a3cb3caaee3, regionState=OPEN, lastHost=0450ab8807f5,33961,1731761492894, regionLocation=0450ab8807f5,33961,1731761492894, openSeqNum=2 2024-11-16T12:51:36,243 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e39334374ca56418de9e14f727ef8576, regionState=OPEN, lastHost=0450ab8807f5,39017,1731761494087, regionLocation=0450ab8807f5,39017,1731761494087, openSeqNum=2 2024-11-16T12:51:36,243 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d3e8566a59cc6153f2f93cef2a417aa, regionState=OPEN, lastHost=0450ab8807f5,32881,1731761492857, regionLocation=0450ab8807f5,32881,1731761492857, openSeqNum=2 2024-11-16T12:51:36,243 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T12:51:36,243 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731761556243 2024-11-16T12:51:36,244 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731761616243 2024-11-16T12:51:36,244 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-16T12:51:36,244 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 0450ab8807f5,39017,1731761494087 had 2 regions 2024-11-16T12:51:36,244 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,44015,1731761492818-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:36,244 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,44015,1731761492818-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:36,244 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,44015,1731761492818-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:36,244 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0450ab8807f5:44015, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:36,244 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:36,245 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:36,245 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=17, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 0450ab8807f5,39017,1731761494087, splitWal=true, meta=false, isMeta: false 2024-11-16T12:51:36,247 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,39017,1731761494087-splitting 2024-11-16T12:51:36,248 INFO [PEWorker-2 {}] master.SplitWALManager(105): 0450ab8807f5,39017,1731761494087 WAL count=1, meta=false 2024-11-16T12:51:36,248 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 0450ab8807f5%2C39017%2C1731761494087.1731761494295}] 2024-11-16T12:51:36,249 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.796sec 2024-11-16T12:51:36,249 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-16T12:51:36,249 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-16T12:51:36,249 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-16T12:51:36,249 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-16T12:51:36,249 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-16T12:51:36,249 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,44015,1731761492818-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T12:51:36,249 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,44015,1731761492818-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-16T12:51:36,250 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=0450ab8807f5,33961,1731761492894 2024-11-16T12:51:36,251 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 0450ab8807f5%2C39017%2C1731761494087.1731761494295, worker=0450ab8807f5,33961,1731761492894}] 2024-11-16T12:51:36,254 WARN [master/0450ab8807f5:0.Chore.1 {}] janitor.CatalogJanitor(177): unknown_server=0450ab8807f5,39017,1731761494087/testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11., unknown_server=0450ab8807f5,39017,1731761494087/testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1731761494194.e39334374ca56418de9e14f727ef8576. 2024-11-16T12:51:36,255 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-16T12:51:36,255 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-16T12:51:36,255 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,44015,1731761492818-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:36,405 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T12:51:36,406 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46407, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T12:51:36,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33961 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-11-16T12:51:36,426 INFO [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,39017,1731761494087-splitting/0450ab8807f5%2C39017%2C1731761494087.1731761494295, size=1.6 K (1682bytes) 2024-11-16T12:51:36,426 INFO [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,39017,1731761494087-splitting/0450ab8807f5%2C39017%2C1731761494087.1731761494295 2024-11-16T12:51:36,427 INFO [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,39017,1731761494087-splitting/0450ab8807f5%2C39017%2C1731761494087.1731761494295 after 1ms 2024-11-16T12:51:36,429 INFO [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,39017,1731761494087-splitting/0450ab8807f5%2C39017%2C1731761494087.1731761494295 took 3ms 2024-11-16T12:51:36,431 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for 9c4c0c2626292bc7607edf03a64b4c11: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "col" sequence_id: 18446744073709551615 } 2024-11-16T12:51:36,432 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for e39334374ca56418de9e14f727ef8576: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "col" sequence_id: 18446744073709551615 } 2024-11-16T12:51:36,433 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,39017,1731761494087-splitting/0450ab8807f5%2C39017%2C1731761494087.1731761494295 so closing down 2024-11-16T12:51:36,433 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T12:51:36,434 INFO [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-0450ab8807f5%2C39017%2C1731761494087.1731761494295.temp 2024-11-16T12:51:36,435 INFO [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/9c4c0c2626292bc7607edf03a64b4c11/recovered.edits/0000000000000000004-0450ab8807f5%2C39017%2C1731761494087.1731761494295.temp 2024-11-16T12:51:36,435 INFO [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T12:51:36,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741907_1085 (size=903) 2024-11-16T12:51:36,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741907_1085 (size=903) 2024-11-16T12:51:36,440 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/9c4c0c2626292bc7607edf03a64b4c11/recovered.edits/0000000000000000004-0450ab8807f5%2C39017%2C1731761494087.1731761494295.temp (wrote 6 edits, skipped 0 edits in 0 ms) 2024-11-16T12:51:36,442 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/9c4c0c2626292bc7607edf03a64b4c11/recovered.edits/0000000000000000004-0450ab8807f5%2C39017%2C1731761494087.1731761494295.temp to hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/9c4c0c2626292bc7607edf03a64b4c11/recovered.edits/0000000000000000009 2024-11-16T12:51:36,442 INFO [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 8 edits across 1 Regions in 13 ms; skipped=2; WAL=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,39017,1731761494087-splitting/0450ab8807f5%2C39017%2C1731761494087.1731761494295, size=1.6 K, length=1682, corrupted=false, cancelled=false 2024-11-16T12:51:36,442 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,39017,1731761494087-splitting/0450ab8807f5%2C39017%2C1731761494087.1731761494295, journal: Splitting hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,39017,1731761494087-splitting/0450ab8807f5%2C39017%2C1731761494087.1731761494295, size=1.6 K (1682bytes) at 1731761496426Finishing writing output for hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,39017,1731761494087-splitting/0450ab8807f5%2C39017%2C1731761494087.1731761494295 so closing down at 1731761496433 (+7 ms)Creating recovered edits writer path=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/9c4c0c2626292bc7607edf03a64b4c11/recovered.edits/0000000000000000004-0450ab8807f5%2C39017%2C1731761494087.1731761494295.temp at 1731761496435 (+2 ms)3 split writer threads finished at 1731761496435Closed recovered edits writer path=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/9c4c0c2626292bc7607edf03a64b4c11/recovered.edits/0000000000000000004-0450ab8807f5%2C39017%2C1731761494087.1731761494295.temp (wrote 6 edits, skipped 0 edits in 0 ms) at 1731761496440 (+5 ms)Rename recovered edits hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/9c4c0c2626292bc7607edf03a64b4c11/recovered.edits/0000000000000000004-0450ab8807f5%2C39017%2C1731761494087.1731761494295.temp to hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/9c4c0c2626292bc7607edf03a64b4c11/recovered.edits/0000000000000000009 at 1731761496442 (+2 ms)Processed 8 edits across 1 Regions in 13 ms; skipped=2; WAL=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,39017,1731761494087-splitting/0450ab8807f5%2C39017%2C1731761494087.1731761494295, size=1.6 K, length=1682, corrupted=false, cancelled=false at 1731761496442 2024-11-16T12:51:36,442 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,39017,1731761494087-splitting/0450ab8807f5%2C39017%2C1731761494087.1731761494295 2024-11-16T12:51:36,442 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-11-16T12:51:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44015 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-11-16T12:51:36,447 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,39017,1731761494087-splitting/0450ab8807f5%2C39017%2C1731761494087.1731761494295 to hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/oldWALs 2024-11-16T12:51:36,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-16T12:51:36,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 0450ab8807f5%2C39017%2C1731761494087.1731761494295, worker=0450ab8807f5,33961,1731761492894 in 196 msec 2024-11-16T12:51:36,451 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=0450ab8807f5,33961,1731761492894 2024-11-16T12:51:36,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-16T12:51:36,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; SplitWALProcedure 0450ab8807f5%2C39017%2C1731761494087.1731761494295, worker=0450ab8807f5,33961,1731761492894 in 203 msec 2024-11-16T12:51:36,455 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,39017,1731761494087-splitting dir is empty, no logs to split. 2024-11-16T12:51:36,455 INFO [PEWorker-2 {}] master.SplitWALManager(105): 0450ab8807f5,39017,1731761494087 WAL count=0, meta=false 2024-11-16T12:51:36,455 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 0450ab8807f5,39017,1731761494087 WAL splitting is done? wals=0, meta=false 2024-11-16T12:51:36,457 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 0450ab8807f5,39017,1731761494087 failed, ignore...File hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,39017,1731761494087-splitting does not exist. 2024-11-16T12:51:36,458 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=e39334374ca56418de9e14f727ef8576, ASSIGN}, {pid=21, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=9c4c0c2626292bc7607edf03a64b4c11, ASSIGN}] 2024-11-16T12:51:36,463 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=9c4c0c2626292bc7607edf03a64b4c11, ASSIGN 2024-11-16T12:51:36,463 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=e39334374ca56418de9e14f727ef8576, ASSIGN 2024-11-16T12:51:36,464 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=9c4c0c2626292bc7607edf03a64b4c11, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T12:51:36,464 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=e39334374ca56418de9e14f727ef8576, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T12:51:36,591 DEBUG [RPCClient-NioEventLoopGroup-6-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testLogSplittingAfterMasterRecoveryDueToZKExpiry', row='', locateType=CURRENT is [region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11., hostname=0450ab8807f5,39017,1731761494087, seqNum=2] 2024-11-16T12:51:36,591 DEBUG [RPCClient-NioEventLoopGroup-6-10 {}] ipc.AbstractRpcClient(357): Not trying to connect to 0450ab8807f5:39017 this server is in the failed servers list 2024-11-16T12:51:36,592 DEBUG [RPCClient-NioEventLoopGroup-6-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11., hostname=0450ab8807f5,39017,1731761494087, seqNum=2 , the old value is region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11., hostname=0450ab8807f5,39017,1731761494087, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=0450ab8807f5:39017 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 0450ab8807f5:39017 2024-11-16T12:51:36,592 DEBUG [RPCClient-NioEventLoopGroup-6-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11., hostname=0450ab8807f5,39017,1731761494087, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 0450ab8807f5:39017 2024-11-16T12:51:36,592 DEBUG [RPCClient-NioEventLoopGroup-6-10 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11., hostname=0450ab8807f5,39017,1731761494087, seqNum=2 from cache 2024-11-16T12:51:36,615 DEBUG [0450ab8807f5:44015 {}] balancer.BalancerClusterState(204): Hosts are {0450ab8807f5=0} racks are {/default-rack=0} 2024-11-16T12:51:36,615 DEBUG [0450ab8807f5:44015 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T12:51:36,615 DEBUG [0450ab8807f5:44015 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T12:51:36,615 DEBUG [0450ab8807f5:44015 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T12:51:36,615 DEBUG [0450ab8807f5:44015 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T12:51:36,615 INFO [0450ab8807f5:44015 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T12:51:36,615 INFO [0450ab8807f5:44015 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T12:51:36,615 DEBUG [0450ab8807f5:44015 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T12:51:36,616 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=9c4c0c2626292bc7607edf03a64b4c11, regionState=OPENING, regionLocation=0450ab8807f5,32881,1731761492857 2024-11-16T12:51:36,616 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=e39334374ca56418de9e14f727ef8576, regionState=OPENING, regionLocation=0450ab8807f5,33961,1731761492894 2024-11-16T12:51:36,618 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-19-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=e39334374ca56418de9e14f727ef8576, ASSIGN because future has completed 2024-11-16T12:51:36,619 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure e39334374ca56418de9e14f727ef8576, server=0450ab8807f5,33961,1731761492894}] 2024-11-16T12:51:36,620 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-19-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=9c4c0c2626292bc7607edf03a64b4c11, ASSIGN because future has completed 2024-11-16T12:51:36,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c4c0c2626292bc7607edf03a64b4c11, server=0450ab8807f5,32881,1731761492857}] 2024-11-16T12:51:36,772 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T12:51:36,774 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48841, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T12:51:36,777 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] handler.AssignRegionHandler(132): Open testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1731761494194.e39334374ca56418de9e14f727ef8576. 2024-11-16T12:51:36,778 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(7752): Opening region: {ENCODED => e39334374ca56418de9e14f727ef8576, NAME => 'testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1731761494194.e39334374ca56418de9e14f727ef8576.', STARTKEY => '4', ENDKEY => '5'} 2024-11-16T12:51:36,778 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testLogSplittingAfterMasterRecoveryDueToZKExpiry e39334374ca56418de9e14f727ef8576 2024-11-16T12:51:36,778 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(898): Instantiated testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1731761494194.e39334374ca56418de9e14f727ef8576.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:36,778 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(7794): checking encryption for e39334374ca56418de9e14f727ef8576 2024-11-16T12:51:36,778 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(7797): checking classloading for e39334374ca56418de9e14f727ef8576 2024-11-16T12:51:36,779 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(132): Open testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11. 2024-11-16T12:51:36,779 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7752): Opening region: {ENCODED => 9c4c0c2626292bc7607edf03a64b4c11, NAME => 'testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11.', STARTKEY => '', ENDKEY => '1'} 2024-11-16T12:51:36,779 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testLogSplittingAfterMasterRecoveryDueToZKExpiry 9c4c0c2626292bc7607edf03a64b4c11 2024-11-16T12:51:36,779 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(898): Instantiated testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:36,779 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7794): checking encryption for 9c4c0c2626292bc7607edf03a64b4c11 2024-11-16T12:51:36,779 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7797): checking classloading for 9c4c0c2626292bc7607edf03a64b4c11 2024-11-16T12:51:36,780 INFO [StoreOpener-e39334374ca56418de9e14f727ef8576-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family col of region e39334374ca56418de9e14f727ef8576 2024-11-16T12:51:36,781 INFO [StoreOpener-9c4c0c2626292bc7607edf03a64b4c11-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family col of region 9c4c0c2626292bc7607edf03a64b4c11 2024-11-16T12:51:36,781 INFO [StoreOpener-e39334374ca56418de9e14f727ef8576-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e39334374ca56418de9e14f727ef8576 columnFamilyName col 2024-11-16T12:51:36,781 DEBUG [StoreOpener-e39334374ca56418de9e14f727ef8576-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:36,781 INFO [StoreOpener-9c4c0c2626292bc7607edf03a64b4c11-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c4c0c2626292bc7607edf03a64b4c11 columnFamilyName col 2024-11-16T12:51:36,781 DEBUG [StoreOpener-9c4c0c2626292bc7607edf03a64b4c11-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:36,782 INFO [StoreOpener-e39334374ca56418de9e14f727ef8576-1 {}] regionserver.HStore(327): Store=e39334374ca56418de9e14f727ef8576/col, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:36,782 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1038): replaying wal for e39334374ca56418de9e14f727ef8576 2024-11-16T12:51:36,782 INFO [StoreOpener-9c4c0c2626292bc7607edf03a64b4c11-1 {}] regionserver.HStore(327): Store=9c4c0c2626292bc7607edf03a64b4c11/col, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:36,782 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1038): replaying wal for 9c4c0c2626292bc7607edf03a64b4c11 2024-11-16T12:51:36,783 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/e39334374ca56418de9e14f727ef8576 2024-11-16T12:51:36,783 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/9c4c0c2626292bc7607edf03a64b4c11 2024-11-16T12:51:36,784 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/e39334374ca56418de9e14f727ef8576 2024-11-16T12:51:36,784 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/9c4c0c2626292bc7607edf03a64b4c11 2024-11-16T12:51:36,784 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1048): stopping wal replay for e39334374ca56418de9e14f727ef8576 2024-11-16T12:51:36,784 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1060): Cleaning up temporary data for e39334374ca56418de9e14f727ef8576 2024-11-16T12:51:36,785 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/9c4c0c2626292bc7607edf03a64b4c11/recovered.edits/0000000000000000009 2024-11-16T12:51:36,786 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1093): writing seq id for e39334374ca56418de9e14f727ef8576 2024-11-16T12:51:36,787 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1114): Opened e39334374ca56418de9e14f727ef8576; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71409387, jitterRate=0.0640827864408493}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T12:51:36,787 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e39334374ca56418de9e14f727ef8576 2024-11-16T12:51:36,787 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1006): Region open journal for e39334374ca56418de9e14f727ef8576: Running coprocessor pre-open hook at 1731761496778Writing region info on filesystem at 1731761496778Initializing all the Stores at 1731761496780 (+2 ms)Instantiating store for column family {NAME => 'col', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761496780Cleaning up temporary data from old regions at 1731761496784 (+4 ms)Running coprocessor post-open hooks at 1731761496787 (+3 ms)Region opened successfully at 1731761496787 2024-11-16T12:51:36,788 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegionServer(2236): Post open deploy tasks for testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1731761494194.e39334374ca56418de9e14f727ef8576., pid=22, masterSystemTime=1731761496772 2024-11-16T12:51:36,788 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5793): Applied 6, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=9, path=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/9c4c0c2626292bc7607edf03a64b4c11/recovered.edits/0000000000000000009 2024-11-16T12:51:36,788 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(2902): Flushing 9c4c0c2626292bc7607edf03a64b4c11 1/1 column families, dataSize=210 B heapSize=928 B 2024-11-16T12:51:36,790 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegionServer(2266): Finished post open deploy task for testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1731761494194.e39334374ca56418de9e14f727ef8576. 2024-11-16T12:51:36,790 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] handler.AssignRegionHandler(153): Opened testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1731761494194.e39334374ca56418de9e14f727ef8576. 2024-11-16T12:51:36,791 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=e39334374ca56418de9e14f727ef8576, regionState=OPEN, openSeqNum=2, regionLocation=0450ab8807f5,33961,1731761492894 2024-11-16T12:51:36,793 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-19-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure e39334374ca56418de9e14f727ef8576, server=0450ab8807f5,33961,1731761492894 because future has completed 2024-11-16T12:51:36,797 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=20 2024-11-16T12:51:36,797 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure e39334374ca56418de9e14f727ef8576, server=0450ab8807f5,33961,1731761492894 in 176 msec 2024-11-16T12:51:36,799 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=17, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=e39334374ca56418de9e14f727ef8576, ASSIGN in 339 msec 2024-11-16T12:51:36,804 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/9c4c0c2626292bc7607edf03a64b4c11/.tmp/col/3bce7ea569af45478a262d3b26ef645d is 39, key is \x00\x00\x00\x00/col:ql/1731761495355/Put/seqid=0 2024-11-16T12:51:36,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741908_1086 (size=5166) 2024-11-16T12:51:36,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741908_1086 (size=5166) 2024-11-16T12:51:36,810 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=210 B at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/9c4c0c2626292bc7607edf03a64b4c11/.tmp/col/3bce7ea569af45478a262d3b26ef645d 2024-11-16T12:51:36,816 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/9c4c0c2626292bc7607edf03a64b4c11/.tmp/col/3bce7ea569af45478a262d3b26ef645d as hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/9c4c0c2626292bc7607edf03a64b4c11/col/3bce7ea569af45478a262d3b26ef645d 2024-11-16T12:51:36,823 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/9c4c0c2626292bc7607edf03a64b4c11/col/3bce7ea569af45478a262d3b26ef645d, entries=6, sequenceid=9, filesize=5.0 K 2024-11-16T12:51:36,823 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(3140): Finished flush of dataSize ~210 B/210, heapSize ~912 B/912, currentSize=0 B/0 for 9c4c0c2626292bc7607edf03a64b4c11 in 35ms, sequenceid=9, compaction requested=false; wal=null 2024-11-16T12:51:36,823 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testLogSplittingAfterMasterRecoveryDueToZKExpiry' 2024-11-16T12:51:36,824 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/9c4c0c2626292bc7607edf03a64b4c11/recovered.edits/0000000000000000009 2024-11-16T12:51:36,825 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1048): stopping wal replay for 9c4c0c2626292bc7607edf03a64b4c11 2024-11-16T12:51:36,825 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1060): Cleaning up temporary data for 9c4c0c2626292bc7607edf03a64b4c11 2024-11-16T12:51:36,827 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1093): writing seq id for 9c4c0c2626292bc7607edf03a64b4c11 2024-11-16T12:51:36,829 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/9c4c0c2626292bc7607edf03a64b4c11/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-16T12:51:36,830 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1114): Opened 9c4c0c2626292bc7607edf03a64b4c11; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61239260, jitterRate=-0.08746391534805298}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T12:51:36,830 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9c4c0c2626292bc7607edf03a64b4c11 2024-11-16T12:51:36,831 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1006): Region open journal for 9c4c0c2626292bc7607edf03a64b4c11: Running coprocessor pre-open hook at 1731761496780Writing region info on filesystem at 1731761496780Initializing all the Stores at 1731761496780Instantiating store for column family {NAME => 'col', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761496780Obtaining lock to block concurrent updates at 1731761496788 (+8 ms)Preparing flush snapshotting stores in 9c4c0c2626292bc7607edf03a64b4c11 at 1731761496788Finished memstore snapshotting testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11., syncing WAL and waiting on mvcc, flushsize=dataSize=210, getHeapSize=912, getOffHeapSize=0, getCellsCount=6 at 1731761496788Flushing stores of testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11. at 1731761496788Flushing 9c4c0c2626292bc7607edf03a64b4c11/col: creating writer at 1731761496788Flushing 9c4c0c2626292bc7607edf03a64b4c11/col: appending metadata at 1731761496804 (+16 ms)Flushing 9c4c0c2626292bc7607edf03a64b4c11/col: closing flushed file at 1731761496804Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@938ca3f: reopening flushed file at 1731761496815 (+11 ms)Finished flush of dataSize ~210 B/210, heapSize ~912 B/912, currentSize=0 B/0 for 9c4c0c2626292bc7607edf03a64b4c11 in 35ms, sequenceid=9, compaction requested=false; wal=null at 1731761496823 (+8 ms)Cleaning up temporary data from old regions at 1731761496825 (+2 ms)Running coprocessor post-open hooks at 1731761496830 (+5 ms)Region opened successfully at 1731761496831 (+1 ms) 2024-11-16T12:51:36,831 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2236): Post open deploy tasks for testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11., pid=23, masterSystemTime=1731761496772 2024-11-16T12:51:36,833 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2266): Finished post open deploy task for testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11. 2024-11-16T12:51:36,834 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(153): Opened testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11. 2024-11-16T12:51:36,834 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=9c4c0c2626292bc7607edf03a64b4c11, regionState=OPEN, openSeqNum=10, regionLocation=0450ab8807f5,32881,1731761492857 2024-11-16T12:51:36,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-19-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c4c0c2626292bc7607edf03a64b4c11, server=0450ab8807f5,32881,1731761492857 because future has completed 2024-11-16T12:51:36,840 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=21 2024-11-16T12:51:36,840 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 9c4c0c2626292bc7607edf03a64b4c11, server=0450ab8807f5,32881,1731761492857 in 218 msec 2024-11-16T12:51:36,842 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=17 2024-11-16T12:51:36,842 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=17, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=9c4c0c2626292bc7607edf03a64b4c11, ASSIGN in 382 msec 2024-11-16T12:51:36,842 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(291): removed crashed server 0450ab8807f5,39017,1731761494087 after splitting done 2024-11-16T12:51:36,843 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; ServerCrashProcedure 0450ab8807f5,39017,1731761494087, splitWal=true, meta=false in 672 msec 2024-11-16T12:51:37,151 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T12:51:37,604 DEBUG [RPCClient-NioEventLoopGroup-6-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testLogSplittingAfterMasterRecoveryDueToZKExpiry', row='', locateType=CURRENT is [region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11., hostname=0450ab8807f5,32881,1731761492857, seqNum=10] 2024-11-16T12:51:37,614 DEBUG [RPCClient-NioEventLoopGroup-6-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testLogSplittingAfterMasterRecoveryDueToZKExpiry', row='1', locateType=CURRENT is [region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1731761494194.ef3a398c9e5522ea013cb58384c3d2e5., hostname=0450ab8807f5,33961,1731761492894, seqNum=2] 2024-11-16T12:51:37,615 DEBUG [RPCClient-NioEventLoopGroup-6-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T12:51:37,617 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-18-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50322, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T12:51:37,621 DEBUG [RPCClient-NioEventLoopGroup-6-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testLogSplittingAfterMasterRecoveryDueToZKExpiry', row='2', locateType=CURRENT is [region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1731761494194.75073183d2c44df3f804b5dab504292d., hostname=0450ab8807f5,32881,1731761492857, seqNum=2] 2024-11-16T12:51:37,626 DEBUG [RPCClient-NioEventLoopGroup-6-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testLogSplittingAfterMasterRecoveryDueToZKExpiry', row='3', locateType=CURRENT is [region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1731761494194.4432ea1606b52a5734457a3cb3caaee3., hostname=0450ab8807f5,33961,1731761492894, seqNum=2] 2024-11-16T12:51:37,631 DEBUG [RPCClient-NioEventLoopGroup-6-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testLogSplittingAfterMasterRecoveryDueToZKExpiry', row='4', locateType=CURRENT is [region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1731761494194.e39334374ca56418de9e14f727ef8576., hostname=0450ab8807f5,33961,1731761492894, seqNum=2] 2024-11-16T12:51:37,636 DEBUG [RPCClient-NioEventLoopGroup-6-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testLogSplittingAfterMasterRecoveryDueToZKExpiry', row='5', locateType=CURRENT is [region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1731761494194.8d3e8566a59cc6153f2f93cef2a417aa., hostname=0450ab8807f5,32881,1731761492857, seqNum=2] 2024-11-16T12:51:37,638 DEBUG [Time-limited test {}] master.HMaster(3213): Abort called but aborted=true, stopped=true 2024-11-16T12:51:37,638 ERROR [Time-limited test {}] master.HMaster(3218): Master server abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-16T12:51:37,638 ERROR [Time-limited test {}] master.HMaster(3225): ***** ABORTING master 0450ab8807f5,44015,1731761492818: killAll ***** 2024-11-16T12:51:37,638 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master '0450ab8807f5,44015,1731761492818' ***** 2024-11-16T12:51:37,638 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: Stopped by Time-limited test 2024-11-16T12:51:37,638 INFO [M:1;0450ab8807f5:44015 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T12:51:37,639 INFO [M:1;0450ab8807f5:44015 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:1;0450ab8807f5:44015. 2024-11-16T12:51:37,639 DEBUG [M:1;0450ab8807f5:44015 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:51:37,639 DEBUG [M:1;0450ab8807f5:44015 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:37,639 INFO [M:1;0450ab8807f5:44015 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T12:51:37,639 DEBUG [M:1;0450ab8807f5:44015 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T12:51:37,639 DEBUG [M:1;0450ab8807f5:44015 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T12:51:37,639 DEBUG [master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.small.0-1731761496236 {}] cleaner.HFileCleaner(306): Exit Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.small.0-1731761496236,5,FailOnTimeoutGroup] 2024-11-16T12:51:37,639 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T12:51:37,639 INFO [M:1;0450ab8807f5:44015 {}] hbase.ChoreService(370): Chore service for: master/0450ab8807f5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T12:51:37,639 DEBUG [master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.large.0-1731761496236 {}] cleaner.HFileCleaner(306): Exit Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.large.0-1731761496236,5,FailOnTimeoutGroup] 2024-11-16T12:51:37,639 INFO [M:1;0450ab8807f5:44015 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T12:51:37,639 DEBUG [M:1;0450ab8807f5:44015 {}] master.HMaster(1795): Stopping service threads 2024-11-16T12:51:37,639 INFO [M:1;0450ab8807f5:44015 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T12:51:37,639 INFO [M:1;0450ab8807f5:44015 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T12:51:37,640 INFO [M:1;0450ab8807f5:44015 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=true 2024-11-16T12:51:37,640 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T12:51:37,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:37,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32881-0x10144fc6ca60014, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:37,686 DEBUG [pool-593-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10144fc6ca60015, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:37,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:37,686 DEBUG [pool-593-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10144fc6ca60015, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:37,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32881-0x10144fc6ca60014, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:37,686 DEBUG [M:1;0450ab8807f5:44015 {}] zookeeper.ZKUtil(347): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-16T12:51:37,686 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 0450ab8807f5,32881,1731761492857: killAll ***** 2024-11-16T12:51:37,686 WARN [M:1;0450ab8807f5:44015 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-16T12:51:37,686 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-16T12:51:37,686 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-16T12:51:37,687 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:32881-0x10144fc6ca60014, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:37,687 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33961-0x10144fc6ca60015, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:37,687 INFO [M:1;0450ab8807f5:44015 {}] master.ServerManager(1134): Rewriting .lastflushedseqids file at: hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/.lastflushedseqids 2024-11-16T12:51:37,687 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-16T12:51:37,688 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-16T12:51:37,688 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-16T12:51:37,689 DEBUG [HMaster-EventLoopGroup-16-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0450ab8807f5,44015,1731761492818 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:37,689 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0450ab8807f5,44015,1731761492818 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:37,690 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 486359984 }, "NonHeapMemoryUsage": { "committed": 168165376, "init": 7667712, "max": -1, "used": 165700432 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "0450ab8807f5", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 35491, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 2, "ProcessCallTime_num_ops": 109, "ProcessCallTime_min": 0, "ProcessCallTime_max": 10, "ProcessCallTime_mean": 1, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 1, "ProcessCallTime_75th_percentile": 2, "ProcessCallTime_90th_percentile": 4, "ProcessCallTime_95th_percentile": 5, "ProcessCallTime_98th_percentile": 10, "ProcessCallTime_99th_percentile": 10, "ProcessCallTime_99.9th_percentile": 10, "ProcessCallTime_TimeRangeCount_0-1": 42, "ProcessCallTime_TimeRangeCount_1-3": 12, "ProcessCallTime_TimeRangeCount_3-10": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 109, "QueueCallTime_min": 0, "QueueCallTime_max": 7, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 1, "QueueCallTime_95th_percentile": 3, "QueueCallTime_98th_percentile": 6, "QueueCallTime_99th_percentile": 7, "QueueCallTime_99.9th_percentile": 7, "QueueCallTime_TimeRangeCount_0-1": 57, "QueueCallTime_TimeRangeCount_1-3": 2, "QueueCallTime_TimeRangeCount_3-10": 3, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 109, "TotalCallTime_min": 0, "TotalCallTime_max": 11, "TotalCallTime_mean": 2, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 1, "TotalCallTime_75th_percentile": 3, "TotalCallTime_90th_percentile": 5, "TotalCallTime_95th_percentile": 8, "TotalCallTime_98th_percentile": 10, "TotalCallTime_99th_percentile": 11, "TotalCallTime_99.9th_percentile": 11, "TotalCallTime_TimeRangeCount_0-1": 37, "TotalCallTime_TimeRangeCount_1-3": 13, "TotalCallTime_TimeRangeCount_3-10": 11, "TotalCallTime_TimeRangeCount_10-30": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 109, "ResponseSize_min": 0, "ResponseSize_max": 6999, "ResponseSize_mean": 468, "ResponseSize_25th_percentile": 1, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 24, "ResponseSize_90th_percentile": 2824, "ResponseSize_95th_percentile": 4911, "ResponseSize_98th_percentile": 6164, "ResponseSize_99th_percentile": 6581, "ResponseSize_99.9th_percentile": 6957, "ResponseSize_SizeRangeCount_0-10": 43, "ResponseSize_SizeRangeCount_10-100": 6, "ResponseSize_SizeRangeCount_100-1000": 13, "exceptions.UnknownScannerException": 0, "exceptions": 2, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 109, "RequestSize_min": 89, "RequestSize_max": 2901, "RequestSize_mean": 268, "RequestSize_25th_percentile": 120, "RequestSize_median": 185, "RequestSize_75th_percentile": 343, "RequestSize_90th_percentile": 1336, "RequestSize_95th_percentile": 2118, "RequestSize_98th_percentile": 2588, "RequestSize_99th_percentile": 2744, "RequestSize_99.9th_percentile": 2885, "RequestSize_SizeRangeCount_10-100": 6, "RequestSize_SizeRangeCount_100-1000": 56, "sentBytes": 28235 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "0450ab8807f5", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:53208", "tag.serverName": "0450ab8807f5,36743,1731761465323", "tag.clusterId": "be09e536-22d9-488d-a733-7968a5bb48e1", "tag.Context": "regionserver", "tag.Hostname": "0450ab8807f5", "regionCount": 0, "storeCount": 0, "hlogFileCount": 11, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731761465323, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 95372, "localBytesRead": 95372, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 105461, "ByteBuffAllocatorPoolAllocationBytes": 66560, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 7, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 1, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 6, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 1, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 6, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 5153, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 74, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 1, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-16T12:51:37,690 DEBUG [RS:1;0450ab8807f5:33961 {}] regionserver.HRegionServer(2589): No master found; retry 2024-11-16T12:51:37,691 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0450ab8807f5,32881,1731761492857' ***** 2024-11-16T12:51:37,691 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: killAll 2024-11-16T12:51:37,691 DEBUG [RS:0;0450ab8807f5:32881 {}] regionserver.HRegionServer(2585): No master found and cluster is stopped; bailing out 2024-11-16T12:51:37,691 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 0450ab8807f5,33961,1731761492894: killAll ***** 2024-11-16T12:51:37,691 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-16T12:51:37,691 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-16T12:51:37,691 INFO [RS:0;0450ab8807f5:32881 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T12:51:37,691 INFO [RS:0;0450ab8807f5:32881 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-16T12:51:37,691 INFO [RS:0;0450ab8807f5:32881 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-16T12:51:37,691 INFO [RS:0;0450ab8807f5:32881 {}] regionserver.HRegionServer(3091): Received CLOSE for 75073183d2c44df3f804b5dab504292d 2024-11-16T12:51:37,691 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-16T12:51:37,691 INFO [RS:0;0450ab8807f5:32881 {}] regionserver.HRegionServer(3091): Received CLOSE for 8d3e8566a59cc6153f2f93cef2a417aa 2024-11-16T12:51:37,691 INFO [RS:0;0450ab8807f5:32881 {}] regionserver.HRegionServer(3091): Received CLOSE for 9c4c0c2626292bc7607edf03a64b4c11 2024-11-16T12:51:37,691 INFO [RS:0;0450ab8807f5:32881 {}] regionserver.HRegionServer(956): aborting server 0450ab8807f5,32881,1731761492857 2024-11-16T12:51:37,691 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T12:51:37,691 INFO [RS:0;0450ab8807f5:32881 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T12:51:37,692 INFO [RS:0;0450ab8807f5:32881 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0450ab8807f5:32881. 2024-11-16T12:51:37,692 DEBUG [RS:0;0450ab8807f5:32881 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:51:37,692 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-16T12:51:37,692 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 75073183d2c44df3f804b5dab504292d, disabling compactions & flushes 2024-11-16T12:51:37,692 DEBUG [RS:0;0450ab8807f5:32881 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:37,692 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1731761494194.75073183d2c44df3f804b5dab504292d. 2024-11-16T12:51:37,692 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1731761494194.75073183d2c44df3f804b5dab504292d. 2024-11-16T12:51:37,692 INFO [RS:0;0450ab8807f5:32881 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T12:51:37,692 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1731761494194.75073183d2c44df3f804b5dab504292d. after waiting 0 ms 2024-11-16T12:51:37,692 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-16T12:51:37,692 INFO [RS:0;0450ab8807f5:32881 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T12:51:37,692 INFO [RS:0;0450ab8807f5:32881 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T12:51:37,692 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1731761494194.75073183d2c44df3f804b5dab504292d. 2024-11-16T12:51:37,692 INFO [RS:0;0450ab8807f5:32881 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T12:51:37,693 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 488457136 }, "NonHeapMemoryUsage": { "committed": 168230912, "init": 7667712, "max": -1, "used": 165732264 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "0450ab8807f5", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 35491, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 2, "ProcessCallTime_num_ops": 109, "ProcessCallTime_min": 0, "ProcessCallTime_max": 10, "ProcessCallTime_mean": 1, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 1, "ProcessCallTime_75th_percentile": 2, "ProcessCallTime_90th_percentile": 4, "ProcessCallTime_95th_percentile": 5, "ProcessCallTime_98th_percentile": 10, "ProcessCallTime_99th_percentile": 10, "ProcessCallTime_99.9th_percentile": 10, "ProcessCallTime_TimeRangeCount_0-1": 42, "ProcessCallTime_TimeRangeCount_1-3": 12, "ProcessCallTime_TimeRangeCount_3-10": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 109, "QueueCallTime_min": 0, "QueueCallTime_max": 7, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 1, "QueueCallTime_95th_percentile": 3, "QueueCallTime_98th_percentile": 6, "QueueCallTime_99th_percentile": 7, "QueueCallTime_99.9th_percentile": 7, "QueueCallTime_TimeRangeCount_0-1": 57, "QueueCallTime_TimeRangeCount_1-3": 2, "QueueCallTime_TimeRangeCount_3-10": 3, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 109, "TotalCallTime_min": 0, "TotalCallTime_max": 11, "TotalCallTime_mean": 2, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 1, "TotalCallTime_75th_percentile": 3, "TotalCallTime_90th_percentile": 5, "TotalCallTime_95th_percentile": 8, "TotalCallTime_98th_percentile": 10, "TotalCallTime_99th_percentile": 11, "TotalCallTime_99.9th_percentile": 11, "TotalCallTime_TimeRangeCount_0-1": 37, "TotalCallTime_TimeRangeCount_1-3": 13, "TotalCallTime_TimeRangeCount_3-10": 11, "TotalCallTime_TimeRangeCount_10-30": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 109, "ResponseSize_min": 0, "ResponseSize_max": 6999, "ResponseSize_mean": 468, "ResponseSize_25th_percentile": 1, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 24, "ResponseSize_90th_percentile": 2824, "ResponseSize_95th_percentile": 4911, "ResponseSize_98th_percentile": 6164, "ResponseSize_99th_percentile": 6581, "ResponseSize_99.9th_percentile": 6957, "ResponseSize_SizeRangeCount_0-10": 43, "ResponseSize_SizeRangeCount_10-100": 6, "ResponseSize_SizeRangeCount_100-1000": 13, "exceptions.UnknownScannerException": 0, "exceptions": 2, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 109, "RequestSize_min": 89, "RequestSize_max": 2901, "RequestSize_mean": 268, "RequestSize_25th_percentile": 120, "RequestSize_median": 185, "RequestSize_75th_percentile": 343, "RequestSize_90th_percentile": 1336, "RequestSize_95th_percentile": 2118, "RequestSize_98th_percentile": 2588, "RequestSize_99th_percentile": 2744, "RequestSize_99.9th_percentile": 2885, "RequestSize_SizeRangeCount_10-100": 6, "RequestSize_SizeRangeCount_100-1000": 56, "sentBytes": 28235 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "0450ab8807f5", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:53208", "tag.serverName": "0450ab8807f5,36743,1731761465323", "tag.clusterId": "be09e536-22d9-488d-a733-7968a5bb48e1", "tag.Context": "regionserver", "tag.Hostname": "0450ab8807f5", "regionCount": 0, "storeCount": 0, "hlogFileCount": 11, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731761465323, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 95372, "localBytesRead": 95372, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 105461, "ByteBuffAllocatorPoolAllocationBytes": 66560, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 7, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 1, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 6, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 1, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 6, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 5153, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 74, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 1, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-16T12:51:37,694 INFO [RS:0;0450ab8807f5:32881 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-11-16T12:51:37,694 DEBUG [RS:0;0450ab8807f5:32881 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 75073183d2c44df3f804b5dab504292d=testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1731761494194.75073183d2c44df3f804b5dab504292d., 8d3e8566a59cc6153f2f93cef2a417aa=testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1731761494194.8d3e8566a59cc6153f2f93cef2a417aa., 9c4c0c2626292bc7607edf03a64b4c11=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11.} 2024-11-16T12:51:37,694 DEBUG [RS:0;0450ab8807f5:32881 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 75073183d2c44df3f804b5dab504292d, 8d3e8566a59cc6153f2f93cef2a417aa, 9c4c0c2626292bc7607edf03a64b4c11 2024-11-16T12:51:37,694 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T12:51:37,694 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T12:51:37,694 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T12:51:37,694 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T12:51:37,694 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T12:51:37,695 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0450ab8807f5,33961,1731761492894' ***** 2024-11-16T12:51:37,695 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: killAll 2024-11-16T12:51:37,695 DEBUG [Time-limited test {}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: killAll 2024-11-16T12:51:37,695 ERROR [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 12622 in region hbase:meta,,1.1588230740 2024-11-16T12:51:37,695 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T12:51:37,695 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T12:51:37,695 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T12:51:37,695 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.TestZooKeeper.after(TestZooKeeper.java:110) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:51:37,695 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731761497694Running coprocessor pre-close hooks at 1731761497694Disabling compacts and flushes for region at 1731761497694Disabling writes for close at 1731761497694Writing region close event to WAL at 1731761497695 (+1 ms)Running coprocessor post-close hooks at 1731761497695Closed at 1731761497695 2024-11-16T12:51:37,695 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:37,695 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T12:51:37,695 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x693ac9cf to 127.0.0.1:53208 2024-11-16T12:51:37,695 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-16T12:51:37,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741909_1087 (size=454) 2024-11-16T12:51:37,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741909_1087 (size=454) 2024-11-16T12:51:37,698 INFO [M:1;0450ab8807f5:44015 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T12:51:37,698 INFO [M:1;0450ab8807f5:44015 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=true 2024-11-16T12:51:37,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741903_1081 (size=30007) 2024-11-16T12:51:37,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741903_1081 (size=30007) 2024-11-16T12:51:37,701 DEBUG [M:1;0450ab8807f5:44015 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T12:51:37,701 INFO [M:1;0450ab8807f5:44015 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:37,701 DEBUG [M:1;0450ab8807f5:44015 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:37,701 DEBUG [M:1;0450ab8807f5:44015 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T12:51:37,701 DEBUG [M:1;0450ab8807f5:44015 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:37,701 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1731761494194.75073183d2c44df3f804b5dab504292d. 2024-11-16T12:51:37,701 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 75073183d2c44df3f804b5dab504292d: Waiting for close lock at 1731761497692Running coprocessor pre-close hooks at 1731761497692Disabling compacts and flushes for region at 1731761497692Disabling writes for close at 1731761497692Writing region close event to WAL at 1731761497701 (+9 ms)Running coprocessor post-close hooks at 1731761497701Closed at 1731761497701 2024-11-16T12:51:37,701 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1731761494194.75073183d2c44df3f804b5dab504292d. 2024-11-16T12:51:37,702 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8d3e8566a59cc6153f2f93cef2a417aa, disabling compactions & flushes 2024-11-16T12:51:37,702 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1731761494194.8d3e8566a59cc6153f2f93cef2a417aa. 2024-11-16T12:51:37,702 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1731761494194.8d3e8566a59cc6153f2f93cef2a417aa. 2024-11-16T12:51:37,702 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1731761494194.8d3e8566a59cc6153f2f93cef2a417aa. after waiting 0 ms 2024-11-16T12:51:37,702 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1731761494194.8d3e8566a59cc6153f2f93cef2a417aa. 2024-11-16T12:51:37,702 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1731761494194.8d3e8566a59cc6153f2f93cef2a417aa. 2024-11-16T12:51:37,702 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8d3e8566a59cc6153f2f93cef2a417aa: Waiting for close lock at 1731761497702Running coprocessor pre-close hooks at 1731761497702Disabling compacts and flushes for region at 1731761497702Disabling writes for close at 1731761497702Writing region close event to WAL at 1731761497702Running coprocessor post-close hooks at 1731761497702Closed at 1731761497702 2024-11-16T12:51:37,702 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1731761494194.8d3e8566a59cc6153f2f93cef2a417aa. 2024-11-16T12:51:37,702 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9c4c0c2626292bc7607edf03a64b4c11, disabling compactions & flushes 2024-11-16T12:51:37,702 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11. 2024-11-16T12:51:37,702 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11. 2024-11-16T12:51:37,702 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11. after waiting 0 ms 2024-11-16T12:51:37,702 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11. 2024-11-16T12:51:37,704 ERROR [M:1;0450ab8807f5:44015 {}] regionserver.HRegion(1960): Memstore data size is 26096 in region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:37,704 INFO [M:1;0450ab8807f5:44015 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:37,704 DEBUG [M:1;0450ab8807f5:44015 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731761497701Disabling compacts and flushes for region at 1731761497701Disabling writes for close at 1731761497701Writing region close event to WAL at 1731761497704 (+3 ms)Closed at 1731761497704 2024-11-16T12:51:37,704 INFO [M:1;0450ab8807f5:44015 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-16T12:51:37,704 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T12:51:37,704 INFO [M:1;0450ab8807f5:44015 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44015 2024-11-16T12:51:37,704 INFO [M:1;0450ab8807f5:44015 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T12:51:37,706 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T12:51:37,707 INFO [regionserver/0450ab8807f5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:51:37,708 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11. 2024-11-16T12:51:37,708 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9c4c0c2626292bc7607edf03a64b4c11: Waiting for close lock at 1731761497702Running coprocessor pre-close hooks at 1731761497702Disabling compacts and flushes for region at 1731761497702Disabling writes for close at 1731761497702Writing region close event to WAL at 1731761497708 (+6 ms)Running coprocessor post-close hooks at 1731761497708Closed at 1731761497708 2024-11-16T12:51:37,708 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1731761494194.9c4c0c2626292bc7607edf03a64b4c11. 2024-11-16T12:51:37,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:37,811 INFO [M:1;0450ab8807f5:44015 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T12:51:37,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44015-0x10144fc6ca60013, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:37,891 INFO [RS:1;0450ab8807f5:33961 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T12:51:37,891 INFO [RS:1;0450ab8807f5:33961 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-16T12:51:37,891 INFO [RS:1;0450ab8807f5:33961 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-16T12:51:37,891 INFO [RS:1;0450ab8807f5:33961 {}] regionserver.HRegionServer(3091): Received CLOSE for 4432ea1606b52a5734457a3cb3caaee3 2024-11-16T12:51:37,891 INFO [RS:1;0450ab8807f5:33961 {}] regionserver.HRegionServer(3091): Received CLOSE for ef3a398c9e5522ea013cb58384c3d2e5 2024-11-16T12:51:37,892 INFO [RS:1;0450ab8807f5:33961 {}] regionserver.HRegionServer(3091): Received CLOSE for e39334374ca56418de9e14f727ef8576 2024-11-16T12:51:37,892 INFO [RS:1;0450ab8807f5:33961 {}] regionserver.HRegionServer(956): aborting server 0450ab8807f5,33961,1731761492894 2024-11-16T12:51:37,892 INFO [RS:1;0450ab8807f5:33961 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T12:51:37,892 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4432ea1606b52a5734457a3cb3caaee3, disabling compactions & flushes 2024-11-16T12:51:37,892 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1731761494194.4432ea1606b52a5734457a3cb3caaee3. 2024-11-16T12:51:37,892 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1731761494194.4432ea1606b52a5734457a3cb3caaee3. 2024-11-16T12:51:37,892 INFO [RS:1;0450ab8807f5:33961 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;0450ab8807f5:33961. 2024-11-16T12:51:37,892 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1731761494194.4432ea1606b52a5734457a3cb3caaee3. after waiting 0 ms 2024-11-16T12:51:37,892 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1731761494194.4432ea1606b52a5734457a3cb3caaee3. 2024-11-16T12:51:37,892 DEBUG [RS:1;0450ab8807f5:33961 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:51:37,892 DEBUG [RS:1;0450ab8807f5:33961 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:37,892 INFO [RS:1;0450ab8807f5:33961 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-11-16T12:51:37,892 DEBUG [RS:1;0450ab8807f5:33961 {}] regionserver.HRegionServer(1325): Online Regions={4432ea1606b52a5734457a3cb3caaee3=testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1731761494194.4432ea1606b52a5734457a3cb3caaee3., ef3a398c9e5522ea013cb58384c3d2e5=testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1731761494194.ef3a398c9e5522ea013cb58384c3d2e5., e39334374ca56418de9e14f727ef8576=testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1731761494194.e39334374ca56418de9e14f727ef8576.} 2024-11-16T12:51:37,892 DEBUG [RS:1;0450ab8807f5:33961 {}] regionserver.HRegionServer(1351): Waiting on 4432ea1606b52a5734457a3cb3caaee3, e39334374ca56418de9e14f727ef8576, ef3a398c9e5522ea013cb58384c3d2e5 2024-11-16T12:51:37,892 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1731761494194.4432ea1606b52a5734457a3cb3caaee3. 2024-11-16T12:51:37,892 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4432ea1606b52a5734457a3cb3caaee3: Waiting for close lock at 1731761497892Running coprocessor pre-close hooks at 1731761497892Disabling compacts and flushes for region at 1731761497892Disabling writes for close at 1731761497892Writing region close event to WAL at 1731761497892Running coprocessor post-close hooks at 1731761497892Closed at 1731761497892 2024-11-16T12:51:37,893 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1731761494194.4432ea1606b52a5734457a3cb3caaee3. 2024-11-16T12:51:37,893 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ef3a398c9e5522ea013cb58384c3d2e5, disabling compactions & flushes 2024-11-16T12:51:37,893 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1731761494194.ef3a398c9e5522ea013cb58384c3d2e5. 2024-11-16T12:51:37,893 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1731761494194.ef3a398c9e5522ea013cb58384c3d2e5. 2024-11-16T12:51:37,893 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1731761494194.ef3a398c9e5522ea013cb58384c3d2e5. after waiting 0 ms 2024-11-16T12:51:37,893 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1731761494194.ef3a398c9e5522ea013cb58384c3d2e5. 2024-11-16T12:51:37,894 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1731761494194.ef3a398c9e5522ea013cb58384c3d2e5. 2024-11-16T12:51:37,894 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ef3a398c9e5522ea013cb58384c3d2e5: Waiting for close lock at 1731761497893Running coprocessor pre-close hooks at 1731761497893Disabling compacts and flushes for region at 1731761497893Disabling writes for close at 1731761497893Writing region close event to WAL at 1731761497894 (+1 ms)Running coprocessor post-close hooks at 1731761497894Closed at 1731761497894 2024-11-16T12:51:37,894 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1731761494194.ef3a398c9e5522ea013cb58384c3d2e5. 2024-11-16T12:51:37,894 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e39334374ca56418de9e14f727ef8576, disabling compactions & flushes 2024-11-16T12:51:37,894 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1731761494194.e39334374ca56418de9e14f727ef8576. 2024-11-16T12:51:37,894 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1731761494194.e39334374ca56418de9e14f727ef8576. 2024-11-16T12:51:37,894 INFO [RS:0;0450ab8807f5:32881 {}] regionserver.HRegionServer(976): stopping server 0450ab8807f5,32881,1731761492857; all regions closed. 2024-11-16T12:51:37,894 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1731761494194.e39334374ca56418de9e14f727ef8576. after waiting 0 ms 2024-11-16T12:51:37,894 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1731761494194.e39334374ca56418de9e14f727ef8576. 2024-11-16T12:51:37,895 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1731761494194.e39334374ca56418de9e14f727ef8576. 2024-11-16T12:51:37,895 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e39334374ca56418de9e14f727ef8576: Waiting for close lock at 1731761497894Running coprocessor pre-close hooks at 1731761497894Disabling compacts and flushes for region at 1731761497894Disabling writes for close at 1731761497894Writing region close event to WAL at 1731761497895 (+1 ms)Running coprocessor post-close hooks at 1731761497895Closed at 1731761497895 2024-11-16T12:51:37,895 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1731761494194.e39334374ca56418de9e14f727ef8576. 2024-11-16T12:51:37,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741892_1070 (size=14032) 2024-11-16T12:51:37,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741892_1070 (size=14032) 2024-11-16T12:51:37,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741891_1069 (size=1296) 2024-11-16T12:51:37,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741891_1069 (size=1296) 2024-11-16T12:51:37,903 DEBUG [RS:0;0450ab8807f5:32881 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:37,903 INFO [RS:0;0450ab8807f5:32881 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:51:37,903 INFO [RS:0;0450ab8807f5:32881 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T12:51:37,903 INFO [RS:0;0450ab8807f5:32881 {}] hbase.ChoreService(370): Chore service for: regionserver/0450ab8807f5:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-16T12:51:37,903 INFO [RS:0;0450ab8807f5:32881 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T12:51:37,903 INFO [regionserver/0450ab8807f5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T12:51:37,904 INFO [RS:0;0450ab8807f5:32881 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32881 2024-11-16T12:51:37,907 INFO [regionserver/0450ab8807f5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:51:37,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32881-0x10144fc6ca60014, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0450ab8807f5,32881,1731761492857 2024-11-16T12:51:37,926 INFO [RS:0;0450ab8807f5:32881 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T12:51:38,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32881-0x10144fc6ca60014, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:38,037 INFO [RS:0;0450ab8807f5:32881 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T12:51:38,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32881-0x10144fc6ca60014, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:38,037 INFO [RS:0;0450ab8807f5:32881 {}] regionserver.HRegionServer(1031): Exiting; stopping=0450ab8807f5,32881,1731761492857; zookeeper connection closed. 2024-11-16T12:51:38,038 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7c18e66f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7c18e66f 2024-11-16T12:51:38,092 INFO [RS:1;0450ab8807f5:33961 {}] regionserver.HRegionServer(976): stopping server 0450ab8807f5,33961,1731761492894; all regions closed. 2024-11-16T12:51:38,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741890_1068 (size=1263) 2024-11-16T12:51:38,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741890_1068 (size=1263) 2024-11-16T12:51:38,095 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,33961,1731761492894/0450ab8807f5%2C33961%2C1731761492894.1731761493535 not finished, retry = 0 2024-11-16T12:51:38,196 DEBUG [RS:1;0450ab8807f5:33961 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:38,196 INFO [RS:1;0450ab8807f5:33961 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:51:38,196 INFO [RS:1;0450ab8807f5:33961 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T12:51:38,197 INFO [RS:1;0450ab8807f5:33961 {}] hbase.ChoreService(370): Chore service for: regionserver/0450ab8807f5:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T12:51:38,197 INFO [RS:1;0450ab8807f5:33961 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T12:51:38,197 INFO [regionserver/0450ab8807f5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T12:51:38,197 INFO [RS:1;0450ab8807f5:33961 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T12:51:38,197 INFO [RS:1;0450ab8807f5:33961 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T12:51:38,197 INFO [RS:1;0450ab8807f5:33961 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T12:51:38,197 INFO [RS:1;0450ab8807f5:33961 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33961 2024-11-16T12:51:38,226 DEBUG [pool-593-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10144fc6ca60015, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0450ab8807f5,33961,1731761492894 2024-11-16T12:51:38,226 INFO [RS:1;0450ab8807f5:33961 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T12:51:38,336 INFO [RS:1;0450ab8807f5:33961 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T12:51:38,336 DEBUG [pool-593-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10144fc6ca60015, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:38,336 INFO [RS:1;0450ab8807f5:33961 {}] regionserver.HRegionServer(1031): Exiting; stopping=0450ab8807f5,33961,1731761492894; zookeeper connection closed. 2024-11-16T12:51:38,336 DEBUG [pool-593-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10144fc6ca60015, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:38,336 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6f6ec3b3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6f6ec3b3 2024-11-16T12:51:38,337 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 2 master(s) and 3 regionserver(s) complete 2024-11-16T12:51:38,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60011, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:38,468 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:38,472 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=testing utility connecting to ZooKeeper ensemble=127.0.0.1:53208 2024-11-16T12:51:38,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility0x0, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:51:38,502 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): testing utility-0x10144fc6ca60018 connected 2024-11-16T12:51:38,502 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-16T12:51:38,502 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/meta-region-server 2024-11-16T12:51:38,503 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/rs 2024-11-16T12:51:38,503 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/splitWAL 2024-11-16T12:51:38,503 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters 2024-11-16T12:51:38,504 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc 2024-11-16T12:51:38,504 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/draining 2024-11-16T12:51:38,504 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/master-maintenance 2024-11-16T12:51:38,505 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot 2024-11-16T12:51:38,505 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/hbaseid 2024-11-16T12:51:38,505 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/table 2024-11-16T12:51:38,507 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc/abort 2024-11-16T12:51:38,508 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc/acquired 2024-11-16T12:51:38,508 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc/reached 2024-11-16T12:51:38,509 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot/abort 2024-11-16T12:51:38,510 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot/acquired 2024-11-16T12:51:38,510 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot/reached 2024-11-16T12:51:38,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/reached 2024-11-16T12:51:38,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/online-snapshot 2024-11-16T12:51:38,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/acquired 2024-11-16T12:51:38,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/abort 2024-11-16T12:51:38,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/reached 2024-11-16T12:51:38,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/flush-table-proc 2024-11-16T12:51:38,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/acquired 2024-11-16T12:51:38,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/abort 2024-11-16T12:51:38,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/table 2024-11-16T12:51:38,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:38,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/hbaseid 2024-11-16T12:51:38,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot 2024-11-16T12:51:38,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master-maintenance 2024-11-16T12:51:38,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/draining 2024-11-16T12:51:38,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc 2024-11-16T12:51:38,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:38,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/splitWAL 2024-11-16T12:51:38,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs 2024-11-16T12:51:38,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:38,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T12:51:38,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase 2024-11-16T12:51:38,533 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestZooKeeper#testLogSplittingAfterMasterRecoveryDueToZKExpiry Thread=403 (was 358) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-16-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-16-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1205652929) connection to localhost/127.0.0.1:44975 from jenkins.hfs.8 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-18-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-16-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/0450ab8807f5:0.logRoller java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:180) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/0450ab8807f5:0.Chore.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: BootstrapNodeManager java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:44975 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1005490740_22 at /127.0.0.1:36028 [Waiting for operation #41] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-18-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.8@localhost:44975 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-15-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-19-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-663311594_22 at /127.0.0.1:48598 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1205652929) connection to localhost/127.0.0.1:44975 from jenkins.hfs.6 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: HMaster-EventLoopGroup-15-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:44975 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-15-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-19-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-19-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-18-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1205652929) connection to localhost/127.0.0.1:44975 from jenkins.hfs.7 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-663311594_22 at /127.0.0.1:50302 [Waiting for operation #53] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=687 (was 630) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=226 (was 237), ProcessCount=11 (was 11), AvailableMemoryMB=3166 (was 3284) 2024-11-16T12:51:38,544 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestZooKeeper#testRegionAssignmentAfterMasterRecoveryDueToZKExpiry Thread=403, OpenFileDescriptor=687, MaxFileDescriptor=1048576, SystemLoadAverage=226, ProcessCount=11, AvailableMemoryMB=3165 2024-11-16T12:51:38,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741910_1088 (size=7) 2024-11-16T12:51:38,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741910_1088 (size=7) 2024-11-16T12:51:38,552 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b with version=8 2024-11-16T12:51:38,552 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/hbase-staging 2024-11-16T12:51:38,553 INFO [Time-limited test {}] client.ConnectionUtils(128): master/0450ab8807f5:0 server-side Connection retries=45 2024-11-16T12:51:38,554 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:38,554 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:38,554 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T12:51:38,554 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:38,554 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T12:51:38,554 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T12:51:38,554 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T12:51:38,554 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39327 2024-11-16T12:51:38,555 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39327 connecting to ZooKeeper ensemble=127.0.0.1:53208 2024-11-16T12:51:38,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:393270x0, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:51:38,562 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39327-0x10144fc6ca60019 connected 2024-11-16T12:51:38,619 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:38,621 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:38,623 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:51:38,623 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b, hbase.cluster.distributed=false 2024-11-16T12:51:38,624 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T12:51:38,626 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39327 2024-11-16T12:51:38,627 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39327 2024-11-16T12:51:38,628 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39327 2024-11-16T12:51:38,634 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39327 2024-11-16T12:51:38,635 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39327 2024-11-16T12:51:38,636 INFO [Time-limited test {}] client.ConnectionUtils(128): master/0450ab8807f5:0 server-side Connection retries=45 2024-11-16T12:51:38,636 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:38,636 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:38,636 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T12:51:38,636 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:38,636 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T12:51:38,636 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T12:51:38,636 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T12:51:38,637 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43165 2024-11-16T12:51:38,638 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43165 connecting to ZooKeeper ensemble=127.0.0.1:53208 2024-11-16T12:51:38,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:431650x0, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:51:38,645 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43165-0x10144fc6ca6001a connected 2024-11-16T12:51:38,655 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:38,656 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:38,658 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43165-0x10144fc6ca6001a, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:51:38,658 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b, hbase.cluster.distributed=false 2024-11-16T12:51:38,659 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43165-0x10144fc6ca6001a, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T12:51:38,660 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43165 2024-11-16T12:51:38,660 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43165 2024-11-16T12:51:38,662 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43165 2024-11-16T12:51:38,664 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43165 2024-11-16T12:51:38,665 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43165 2024-11-16T12:51:38,685 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0450ab8807f5:0 server-side Connection retries=45 2024-11-16T12:51:38,685 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:38,685 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:38,685 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T12:51:38,685 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:38,685 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T12:51:38,685 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T12:51:38,685 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T12:51:38,686 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45113 2024-11-16T12:51:38,687 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45113 connecting to ZooKeeper ensemble=127.0.0.1:53208 2024-11-16T12:51:38,688 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:38,690 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:38,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:451130x0, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:51:38,703 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:451130x0, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:51:38,703 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45113-0x10144fc6ca6001b connected 2024-11-16T12:51:38,704 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T12:51:38,704 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T12:51:38,705 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45113-0x10144fc6ca6001b, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:38,705 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45113-0x10144fc6ca6001b, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T12:51:38,706 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45113 2024-11-16T12:51:38,706 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45113 2024-11-16T12:51:38,706 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45113 2024-11-16T12:51:38,707 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45113 2024-11-16T12:51:38,709 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45113 2024-11-16T12:51:38,722 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0450ab8807f5:0 server-side Connection retries=45 2024-11-16T12:51:38,723 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:38,723 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:38,723 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T12:51:38,723 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:38,723 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T12:51:38,723 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T12:51:38,723 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T12:51:38,724 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46757 2024-11-16T12:51:38,725 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46757 connecting to ZooKeeper ensemble=127.0.0.1:53208 2024-11-16T12:51:38,726 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:38,727 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:38,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:467570x0, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:51:38,737 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:467570x0, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:51:38,737 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46757-0x10144fc6ca6001c connected 2024-11-16T12:51:38,737 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T12:51:38,737 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T12:51:38,738 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46757-0x10144fc6ca6001c, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:38,738 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46757-0x10144fc6ca6001c, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T12:51:38,742 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46757 2024-11-16T12:51:38,743 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46757 2024-11-16T12:51:38,745 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46757 2024-11-16T12:51:38,746 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46757 2024-11-16T12:51:38,746 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46757 2024-11-16T12:51:38,758 DEBUG [M:0;0450ab8807f5:39327 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0450ab8807f5:39327 2024-11-16T12:51:38,758 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0450ab8807f5,39327,1731761498553 2024-11-16T12:51:38,761 DEBUG [M:1;0450ab8807f5:43165 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:1;0450ab8807f5:43165 2024-11-16T12:51:38,762 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0450ab8807f5,43165,1731761498636 2024-11-16T12:51:38,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x10144fc6ca6001a, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:38,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x10144fc6ca6001c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:38,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:38,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45113-0x10144fc6ca6001b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:38,778 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0450ab8807f5,39327,1731761498553 2024-11-16T12:51:38,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x10144fc6ca6001a, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:38,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:38,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x10144fc6ca6001c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:38,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45113-0x10144fc6ca6001b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:38,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x10144fc6ca6001c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:38,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45113-0x10144fc6ca6001b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:38,786 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43165-0x10144fc6ca6001a, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0450ab8807f5,43165,1731761498636 2024-11-16T12:51:38,794 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T12:51:38,794 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43165-0x10144fc6ca6001a, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T12:51:38,794 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0450ab8807f5,39327,1731761498553 from backup master directory 2024-11-16T12:51:38,795 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ActiveMasterManager(296): Another master is the active master, 0450ab8807f5,39327,1731761498553; waiting to become the next active master 2024-11-16T12:51:38,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45113-0x10144fc6ca6001b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:38,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x10144fc6ca6001c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:38,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0450ab8807f5,39327,1731761498553 2024-11-16T12:51:38,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x10144fc6ca6001a, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:38,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:38,803 WARN [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T12:51:38,803 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0450ab8807f5,39327,1731761498553 2024-11-16T12:51:38,807 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/hbase.id] with ID: 143d46ab-afa8-4557-97d2-293a8ded4462 2024-11-16T12:51:38,807 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/.tmp/hbase.id 2024-11-16T12:51:38,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741911_1089 (size=42) 2024-11-16T12:51:38,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741911_1089 (size=42) 2024-11-16T12:51:38,812 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/.tmp/hbase.id]:[hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/hbase.id] 2024-11-16T12:51:38,823 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:38,823 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T12:51:38,824 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-16T12:51:38,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:38,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x10144fc6ca6001c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:38,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x10144fc6ca6001a, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:38,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45113-0x10144fc6ca6001b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:38,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741912_1090 (size=196) 2024-11-16T12:51:38,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741912_1090 (size=196) 2024-11-16T12:51:38,842 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T12:51:38,842 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T12:51:38,843 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:51:38,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741913_1091 (size=1189) 2024-11-16T12:51:38,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741913_1091 (size=1189) 2024-11-16T12:51:38,852 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store 2024-11-16T12:51:38,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741914_1092 (size=34) 2024-11-16T12:51:38,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741914_1092 (size=34) 2024-11-16T12:51:38,859 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:38,859 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T12:51:38,859 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:38,859 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:38,859 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T12:51:38,859 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:38,859 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:38,859 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731761498859Disabling compacts and flushes for region at 1731761498859Disabling writes for close at 1731761498859Writing region close event to WAL at 1731761498859Closed at 1731761498859 2024-11-16T12:51:38,860 WARN [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/.initializing 2024-11-16T12:51:38,860 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,39327,1731761498553 2024-11-16T12:51:38,861 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T12:51:38,863 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0450ab8807f5%2C39327%2C1731761498553, suffix=, logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,39327,1731761498553, archiveDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/oldWALs, maxLogs=10 2024-11-16T12:51:38,875 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,39327,1731761498553/0450ab8807f5%2C39327%2C1731761498553.1731761498863, exclude list is [], retry=0 2024-11-16T12:51:38,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43257,DS-06d91055-02a4-40d7-bddb-15aee81f02ac,DISK] 2024-11-16T12:51:38,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36961,DS-7ef9e9a6-250e-467c-989b-a9ffcab2299e,DISK] 2024-11-16T12:51:38,879 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,39327,1731761498553/0450ab8807f5%2C39327%2C1731761498553.1731761498863 2024-11-16T12:51:38,879 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:46667:46667)] 2024-11-16T12:51:38,879 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-16T12:51:38,879 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:38,879 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:38,879 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:38,881 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:38,882 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-16T12:51:38,882 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:38,883 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:38,883 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:38,884 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-16T12:51:38,884 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:38,885 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:38,885 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:38,886 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-16T12:51:38,886 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:38,886 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:38,886 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:38,888 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-16T12:51:38,888 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:38,888 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:38,888 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:38,889 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:38,889 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:38,890 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:38,890 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:38,891 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T12:51:38,892 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:51:38,895 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:51:38,895 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65409932, jitterRate=-0.025316059589385986}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T12:51:38,896 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731761498879Initializing all the Stores at 1731761498880 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761498880Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761498880Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761498881 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761498881Cleaning up temporary data from old regions at 1731761498890 (+9 ms)Region opened successfully at 1731761498895 (+5 ms) 2024-11-16T12:51:38,896 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-16T12:51:38,899 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@445fc028, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0450ab8807f5/172.17.0.2:0 2024-11-16T12:51:38,900 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-16T12:51:38,900 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-16T12:51:38,900 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-16T12:51:38,901 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T12:51:38,901 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T12:51:38,901 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-16T12:51:38,901 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-16T12:51:38,903 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-16T12:51:38,904 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-16T12:51:38,918 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-16T12:51:38,918 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-16T12:51:38,919 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-16T12:51:38,928 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-16T12:51:38,928 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-16T12:51:38,929 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-16T12:51:38,936 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-16T12:51:38,937 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-16T12:51:38,944 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-16T12:51:38,946 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-16T12:51:38,952 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-16T12:51:38,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x10144fc6ca6001c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T12:51:38,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45113-0x10144fc6ca6001b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T12:51:38,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x10144fc6ca6001a, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T12:51:38,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T12:51:38,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x10144fc6ca6001a, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:38,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x10144fc6ca6001c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:38,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45113-0x10144fc6ca6001b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:38,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:38,983 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0450ab8807f5,39327,1731761498553, sessionid=0x10144fc6ca60019, setting cluster-up flag (Was=false) 2024-11-16T12:51:39,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x10144fc6ca6001a, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:39,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45113-0x10144fc6ca6001b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:39,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:39,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x10144fc6ca6001c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:39,036 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-16T12:51:39,037 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0450ab8807f5,39327,1731761498553 2024-11-16T12:51:39,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:39,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x10144fc6ca6001a, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:39,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x10144fc6ca6001c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:39,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45113-0x10144fc6ca6001b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:39,078 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-16T12:51:39,079 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0450ab8807f5,39327,1731761498553 2024-11-16T12:51:39,080 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-16T12:51:39,081 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T12:51:39,081 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-16T12:51:39,081 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0450ab8807f5,39327,1731761498553 Number of backup masters: 1 0450ab8807f5,43165,1731761498636 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-16T12:51:39,082 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:39,083 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:39,083 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:39,083 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:51:39,083 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0450ab8807f5:0, corePoolSize=10, maxPoolSize=10 2024-11-16T12:51:39,083 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,083 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0450ab8807f5:0, corePoolSize=2, maxPoolSize=2 2024-11-16T12:51:39,083 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,084 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T12:51:39,084 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-16T12:51:39,084 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731761529084 2024-11-16T12:51:39,085 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-16T12:51:39,085 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-16T12:51:39,085 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-16T12:51:39,085 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-16T12:51:39,085 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-16T12:51:39,085 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-16T12:51:39,085 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,085 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-16T12:51:39,085 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:39,085 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-16T12:51:39,085 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-16T12:51:39,085 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-16T12:51:39,086 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-16T12:51:39,086 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-16T12:51:39,086 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.large.0-1731761499086,5,FailOnTimeoutGroup] 2024-11-16T12:51:39,089 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.small.0-1731761499086,5,FailOnTimeoutGroup] 2024-11-16T12:51:39,089 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,089 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-16T12:51:39,089 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,089 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741916_1094 (size=1321) 2024-11-16T12:51:39,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741916_1094 (size=1321) 2024-11-16T12:51:39,098 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-16T12:51:39,098 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:39,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741917_1095 (size=32) 2024-11-16T12:51:39,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741917_1095 (size=32) 2024-11-16T12:51:39,109 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:39,110 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T12:51:39,112 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T12:51:39,112 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:39,112 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:39,113 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T12:51:39,114 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T12:51:39,114 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:39,114 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:39,114 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T12:51:39,116 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T12:51:39,116 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:39,116 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:39,116 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T12:51:39,117 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T12:51:39,117 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:39,118 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:39,118 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T12:51:39,119 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740 2024-11-16T12:51:39,119 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740 2024-11-16T12:51:39,120 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T12:51:39,120 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T12:51:39,121 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T12:51:39,122 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T12:51:39,124 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:51:39,124 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62535542, jitterRate=-0.06814780831336975}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T12:51:39,125 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731761499109Initializing all the Stores at 1731761499110 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761499110Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761499110Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761499110Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761499110Cleaning up temporary data from old regions at 1731761499120 (+10 ms)Region opened successfully at 1731761499125 (+5 ms) 2024-11-16T12:51:39,125 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T12:51:39,125 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T12:51:39,125 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T12:51:39,125 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T12:51:39,125 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T12:51:39,126 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T12:51:39,126 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731761499125Disabling compacts and flushes for region at 1731761499125Disabling writes for close at 1731761499125Writing region close event to WAL at 1731761499126 (+1 ms)Closed at 1731761499126 2024-11-16T12:51:39,127 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T12:51:39,127 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-16T12:51:39,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T12:51:39,128 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T12:51:39,129 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-16T12:51:39,149 INFO [RS:1;0450ab8807f5:46757 {}] regionserver.HRegionServer(746): ClusterId : 143d46ab-afa8-4557-97d2-293a8ded4462 2024-11-16T12:51:39,149 INFO [RS:0;0450ab8807f5:45113 {}] regionserver.HRegionServer(746): ClusterId : 143d46ab-afa8-4557-97d2-293a8ded4462 2024-11-16T12:51:39,149 DEBUG [RS:1;0450ab8807f5:46757 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T12:51:39,149 DEBUG [RS:0;0450ab8807f5:45113 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T12:51:39,179 DEBUG [RS:1;0450ab8807f5:46757 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T12:51:39,179 DEBUG [RS:0;0450ab8807f5:45113 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T12:51:39,179 DEBUG [RS:1;0450ab8807f5:46757 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T12:51:39,179 DEBUG [RS:0;0450ab8807f5:45113 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T12:51:39,187 DEBUG [RS:1;0450ab8807f5:46757 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T12:51:39,187 DEBUG [RS:0;0450ab8807f5:45113 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T12:51:39,187 DEBUG [RS:1;0450ab8807f5:46757 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52857927, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0450ab8807f5/172.17.0.2:0 2024-11-16T12:51:39,187 DEBUG [RS:0;0450ab8807f5:45113 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3128ffe2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0450ab8807f5/172.17.0.2:0 2024-11-16T12:51:39,197 DEBUG [RS:1;0450ab8807f5:46757 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;0450ab8807f5:46757 2024-11-16T12:51:39,198 INFO [RS:1;0450ab8807f5:46757 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T12:51:39,198 INFO [RS:1;0450ab8807f5:46757 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T12:51:39,198 DEBUG [RS:1;0450ab8807f5:46757 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T12:51:39,198 INFO [RS:1;0450ab8807f5:46757 {}] regionserver.HRegionServer(2659): reportForDuty to master=0450ab8807f5,39327,1731761498553 with port=46757, startcode=1731761498722 2024-11-16T12:51:39,198 DEBUG [RS:1;0450ab8807f5:46757 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T12:51:39,200 INFO [HMaster-EventLoopGroup-20-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47325, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.10 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T12:51:39,200 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39327 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0450ab8807f5,46757,1731761498722 2024-11-16T12:51:39,200 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39327 {}] master.ServerManager(517): Registering regionserver=0450ab8807f5,46757,1731761498722 2024-11-16T12:51:39,201 DEBUG [RS:0;0450ab8807f5:45113 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0450ab8807f5:45113 2024-11-16T12:51:39,201 INFO [RS:0;0450ab8807f5:45113 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T12:51:39,201 INFO [RS:0;0450ab8807f5:45113 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T12:51:39,201 DEBUG [RS:0;0450ab8807f5:45113 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T12:51:39,202 DEBUG [RS:1;0450ab8807f5:46757 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:39,202 DEBUG [RS:1;0450ab8807f5:46757 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44975 2024-11-16T12:51:39,202 DEBUG [RS:1;0450ab8807f5:46757 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T12:51:39,202 INFO [RS:0;0450ab8807f5:45113 {}] regionserver.HRegionServer(2659): reportForDuty to master=0450ab8807f5,39327,1731761498553 with port=45113, startcode=1731761498684 2024-11-16T12:51:39,202 DEBUG [RS:0;0450ab8807f5:45113 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T12:51:39,204 INFO [HMaster-EventLoopGroup-20-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54765, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.9 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T12:51:39,204 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39327 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0450ab8807f5,45113,1731761498684 2024-11-16T12:51:39,204 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39327 {}] master.ServerManager(517): Registering regionserver=0450ab8807f5,45113,1731761498684 2024-11-16T12:51:39,205 DEBUG [RS:0;0450ab8807f5:45113 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:39,205 DEBUG [RS:0;0450ab8807f5:45113 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44975 2024-11-16T12:51:39,205 DEBUG [RS:0;0450ab8807f5:45113 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T12:51:39,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T12:51:39,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x10144fc6ca6001a, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T12:51:39,219 DEBUG [RS:1;0450ab8807f5:46757 {}] zookeeper.ZKUtil(111): regionserver:46757-0x10144fc6ca6001c, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0450ab8807f5,46757,1731761498722 2024-11-16T12:51:39,219 WARN [RS:1;0450ab8807f5:46757 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T12:51:39,219 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0450ab8807f5,45113,1731761498684] 2024-11-16T12:51:39,219 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0450ab8807f5,46757,1731761498722] 2024-11-16T12:51:39,220 INFO [RS:1;0450ab8807f5:46757 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:51:39,220 DEBUG [RS:1;0450ab8807f5:46757 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,46757,1731761498722 2024-11-16T12:51:39,220 DEBUG [RS:0;0450ab8807f5:45113 {}] zookeeper.ZKUtil(111): regionserver:45113-0x10144fc6ca6001b, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0450ab8807f5,45113,1731761498684 2024-11-16T12:51:39,220 WARN [RS:0;0450ab8807f5:45113 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T12:51:39,220 INFO [RS:0;0450ab8807f5:45113 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:51:39,220 DEBUG [RS:0;0450ab8807f5:45113 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,45113,1731761498684 2024-11-16T12:51:39,223 INFO [RS:1;0450ab8807f5:46757 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T12:51:39,223 INFO [RS:0;0450ab8807f5:45113 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T12:51:39,225 INFO [RS:1;0450ab8807f5:46757 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T12:51:39,225 INFO [RS:1;0450ab8807f5:46757 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T12:51:39,225 INFO [RS:1;0450ab8807f5:46757 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,225 INFO [RS:1;0450ab8807f5:46757 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T12:51:39,226 INFO [RS:1;0450ab8807f5:46757 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T12:51:39,226 INFO [RS:0;0450ab8807f5:45113 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T12:51:39,227 INFO [RS:1;0450ab8807f5:46757 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,227 DEBUG [RS:1;0450ab8807f5:46757 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,227 DEBUG [RS:1;0450ab8807f5:46757 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,227 DEBUG [RS:1;0450ab8807f5:46757 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,227 DEBUG [RS:1;0450ab8807f5:46757 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,227 DEBUG [RS:1;0450ab8807f5:46757 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,227 DEBUG [RS:1;0450ab8807f5:46757 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0, corePoolSize=2, maxPoolSize=2 2024-11-16T12:51:39,227 DEBUG [RS:1;0450ab8807f5:46757 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,227 DEBUG [RS:1;0450ab8807f5:46757 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,227 DEBUG [RS:1;0450ab8807f5:46757 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,227 DEBUG [RS:1;0450ab8807f5:46757 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,227 DEBUG [RS:1;0450ab8807f5:46757 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,227 DEBUG [RS:1;0450ab8807f5:46757 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,227 DEBUG [RS:1;0450ab8807f5:46757 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:51:39,227 DEBUG [RS:1;0450ab8807f5:46757 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:51:39,230 INFO [RS:0;0450ab8807f5:45113 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T12:51:39,230 INFO [RS:0;0450ab8807f5:45113 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,230 INFO [RS:0;0450ab8807f5:45113 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T12:51:39,231 INFO [RS:0;0450ab8807f5:45113 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T12:51:39,231 INFO [RS:0;0450ab8807f5:45113 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,232 DEBUG [RS:0;0450ab8807f5:45113 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,232 DEBUG [RS:0;0450ab8807f5:45113 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,232 DEBUG [RS:0;0450ab8807f5:45113 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,232 DEBUG [RS:0;0450ab8807f5:45113 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,232 DEBUG [RS:0;0450ab8807f5:45113 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,232 DEBUG [RS:0;0450ab8807f5:45113 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0, corePoolSize=2, maxPoolSize=2 2024-11-16T12:51:39,232 DEBUG [RS:0;0450ab8807f5:45113 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,232 DEBUG [RS:0;0450ab8807f5:45113 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,232 DEBUG [RS:0;0450ab8807f5:45113 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,232 DEBUG [RS:0;0450ab8807f5:45113 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,232 DEBUG [RS:0;0450ab8807f5:45113 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,232 DEBUG [RS:0;0450ab8807f5:45113 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,232 DEBUG [RS:0;0450ab8807f5:45113 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:51:39,232 DEBUG [RS:0;0450ab8807f5:45113 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:51:39,236 INFO [RS:1;0450ab8807f5:46757 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,236 INFO [RS:1;0450ab8807f5:46757 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,236 INFO [RS:1;0450ab8807f5:46757 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,236 INFO [RS:1;0450ab8807f5:46757 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,236 INFO [RS:1;0450ab8807f5:46757 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,236 INFO [RS:1;0450ab8807f5:46757 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,46757,1731761498722-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T12:51:39,238 INFO [RS:0;0450ab8807f5:45113 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,238 INFO [RS:0;0450ab8807f5:45113 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,238 INFO [RS:0;0450ab8807f5:45113 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,238 INFO [RS:0;0450ab8807f5:45113 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,238 INFO [RS:0;0450ab8807f5:45113 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,238 INFO [RS:0;0450ab8807f5:45113 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,45113,1731761498684-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T12:51:39,252 INFO [RS:1;0450ab8807f5:46757 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T12:51:39,253 INFO [RS:1;0450ab8807f5:46757 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,46757,1731761498722-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,253 INFO [RS:1;0450ab8807f5:46757 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,253 INFO [RS:1;0450ab8807f5:46757 {}] regionserver.Replication(171): 0450ab8807f5,46757,1731761498722 started 2024-11-16T12:51:39,258 INFO [RS:0;0450ab8807f5:45113 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T12:51:39,258 INFO [RS:0;0450ab8807f5:45113 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,45113,1731761498684-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,258 INFO [RS:0;0450ab8807f5:45113 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,258 INFO [RS:0;0450ab8807f5:45113 {}] regionserver.Replication(171): 0450ab8807f5,45113,1731761498684 started 2024-11-16T12:51:39,266 INFO [RS:1;0450ab8807f5:46757 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,267 INFO [RS:1;0450ab8807f5:46757 {}] regionserver.HRegionServer(1482): Serving as 0450ab8807f5,46757,1731761498722, RpcServer on 0450ab8807f5/172.17.0.2:46757, sessionid=0x10144fc6ca6001c 2024-11-16T12:51:39,267 DEBUG [RS:1;0450ab8807f5:46757 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T12:51:39,267 DEBUG [RS:1;0450ab8807f5:46757 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0450ab8807f5,46757,1731761498722 2024-11-16T12:51:39,267 DEBUG [RS:1;0450ab8807f5:46757 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,46757,1731761498722' 2024-11-16T12:51:39,267 DEBUG [RS:1;0450ab8807f5:46757 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T12:51:39,267 DEBUG [RS:1;0450ab8807f5:46757 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T12:51:39,267 DEBUG [RS:1;0450ab8807f5:46757 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T12:51:39,268 DEBUG [RS:1;0450ab8807f5:46757 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T12:51:39,268 DEBUG [RS:1;0450ab8807f5:46757 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0450ab8807f5,46757,1731761498722 2024-11-16T12:51:39,268 DEBUG [RS:1;0450ab8807f5:46757 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,46757,1731761498722' 2024-11-16T12:51:39,268 DEBUG [RS:1;0450ab8807f5:46757 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T12:51:39,268 DEBUG [RS:1;0450ab8807f5:46757 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T12:51:39,268 DEBUG [RS:1;0450ab8807f5:46757 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T12:51:39,268 INFO [RS:1;0450ab8807f5:46757 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T12:51:39,268 INFO [RS:1;0450ab8807f5:46757 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T12:51:39,278 INFO [RS:0;0450ab8807f5:45113 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,278 INFO [RS:0;0450ab8807f5:45113 {}] regionserver.HRegionServer(1482): Serving as 0450ab8807f5,45113,1731761498684, RpcServer on 0450ab8807f5/172.17.0.2:45113, sessionid=0x10144fc6ca6001b 2024-11-16T12:51:39,278 DEBUG [RS:0;0450ab8807f5:45113 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T12:51:39,278 DEBUG [RS:0;0450ab8807f5:45113 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0450ab8807f5,45113,1731761498684 2024-11-16T12:51:39,278 DEBUG [RS:0;0450ab8807f5:45113 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,45113,1731761498684' 2024-11-16T12:51:39,278 DEBUG [RS:0;0450ab8807f5:45113 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T12:51:39,279 DEBUG [RS:0;0450ab8807f5:45113 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T12:51:39,279 DEBUG [RS:0;0450ab8807f5:45113 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T12:51:39,279 DEBUG [RS:0;0450ab8807f5:45113 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T12:51:39,279 DEBUG [RS:0;0450ab8807f5:45113 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0450ab8807f5,45113,1731761498684 2024-11-16T12:51:39,279 DEBUG [RS:0;0450ab8807f5:45113 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,45113,1731761498684' 2024-11-16T12:51:39,279 DEBUG [RS:0;0450ab8807f5:45113 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T12:51:39,279 WARN [0450ab8807f5:39327 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-16T12:51:39,280 DEBUG [RS:0;0450ab8807f5:45113 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T12:51:39,280 DEBUG [RS:0;0450ab8807f5:45113 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T12:51:39,280 INFO [RS:0;0450ab8807f5:45113 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T12:51:39,280 INFO [RS:0;0450ab8807f5:45113 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T12:51:39,369 INFO [RS:1;0450ab8807f5:46757 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T12:51:39,370 INFO [RS:1;0450ab8807f5:46757 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0450ab8807f5%2C46757%2C1731761498722, suffix=, logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,46757,1731761498722, archiveDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/oldWALs, maxLogs=32 2024-11-16T12:51:39,380 INFO [RS:0;0450ab8807f5:45113 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T12:51:39,382 INFO [RS:0;0450ab8807f5:45113 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0450ab8807f5%2C45113%2C1731761498684, suffix=, logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,45113,1731761498684, archiveDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/oldWALs, maxLogs=32 2024-11-16T12:51:39,384 DEBUG [RS:1;0450ab8807f5:46757 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,46757,1731761498722/0450ab8807f5%2C46757%2C1731761498722.1731761499370, exclude list is [], retry=0 2024-11-16T12:51:39,387 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36961,DS-7ef9e9a6-250e-467c-989b-a9ffcab2299e,DISK] 2024-11-16T12:51:39,387 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43257,DS-06d91055-02a4-40d7-bddb-15aee81f02ac,DISK] 2024-11-16T12:51:39,388 INFO [RS:1;0450ab8807f5:46757 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,46757,1731761498722/0450ab8807f5%2C46757%2C1731761498722.1731761499370 2024-11-16T12:51:39,389 DEBUG [RS:1;0450ab8807f5:46757 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:46667:46667)] 2024-11-16T12:51:39,400 DEBUG [RS:0;0450ab8807f5:45113 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,45113,1731761498684/0450ab8807f5%2C45113%2C1731761498684.1731761499382, exclude list is [], retry=0 2024-11-16T12:51:39,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43257,DS-06d91055-02a4-40d7-bddb-15aee81f02ac,DISK] 2024-11-16T12:51:39,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36961,DS-7ef9e9a6-250e-467c-989b-a9ffcab2299e,DISK] 2024-11-16T12:51:39,405 INFO [RS:0;0450ab8807f5:45113 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,45113,1731761498684/0450ab8807f5%2C45113%2C1731761498684.1731761499382 2024-11-16T12:51:39,406 DEBUG [RS:0;0450ab8807f5:45113 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:46667:46667)] 2024-11-16T12:51:39,530 DEBUG [0450ab8807f5:39327 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-16T12:51:39,530 DEBUG [0450ab8807f5:39327 {}] balancer.BalancerClusterState(204): Hosts are {0450ab8807f5=0} racks are {/default-rack=0} 2024-11-16T12:51:39,532 DEBUG [0450ab8807f5:39327 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T12:51:39,532 DEBUG [0450ab8807f5:39327 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T12:51:39,532 DEBUG [0450ab8807f5:39327 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T12:51:39,532 DEBUG [0450ab8807f5:39327 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T12:51:39,532 INFO [0450ab8807f5:39327 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T12:51:39,532 INFO [0450ab8807f5:39327 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T12:51:39,532 DEBUG [0450ab8807f5:39327 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T12:51:39,532 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0450ab8807f5,45113,1731761498684 2024-11-16T12:51:39,533 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0450ab8807f5,45113,1731761498684, state=OPENING 2024-11-16T12:51:39,568 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-16T12:51:39,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:39,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x10144fc6ca6001a, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:39,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45113-0x10144fc6ca6001b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:39,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x10144fc6ca6001c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:39,578 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:39,578 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:39,578 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:39,578 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:39,578 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T12:51:39,578 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0450ab8807f5,45113,1731761498684}] 2024-11-16T12:51:39,731 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T12:51:39,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-22-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53741, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T12:51:39,736 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T12:51:39,736 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:51:39,736 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T12:51:39,738 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0450ab8807f5%2C45113%2C1731761498684.meta, suffix=.meta, logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,45113,1731761498684, archiveDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/oldWALs, maxLogs=32 2024-11-16T12:51:39,750 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,45113,1731761498684/0450ab8807f5%2C45113%2C1731761498684.meta.1731761499738.meta, exclude list is [], retry=0 2024-11-16T12:51:39,752 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43257,DS-06d91055-02a4-40d7-bddb-15aee81f02ac,DISK] 2024-11-16T12:51:39,752 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36961,DS-7ef9e9a6-250e-467c-989b-a9ffcab2299e,DISK] 2024-11-16T12:51:39,753 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,45113,1731761498684/0450ab8807f5%2C45113%2C1731761498684.meta.1731761499738.meta 2024-11-16T12:51:39,754 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46667:46667),(127.0.0.1/127.0.0.1:38819:38819)] 2024-11-16T12:51:39,754 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T12:51:39,754 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T12:51:39,754 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T12:51:39,754 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T12:51:39,754 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T12:51:39,754 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:39,755 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T12:51:39,755 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T12:51:39,756 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T12:51:39,757 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T12:51:39,757 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:39,758 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:39,758 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T12:51:39,758 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T12:51:39,758 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:39,759 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:39,759 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T12:51:39,760 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T12:51:39,760 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:39,760 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:39,760 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T12:51:39,761 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T12:51:39,761 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:39,761 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:51:39,761 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T12:51:39,762 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740 2024-11-16T12:51:39,763 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/hbase/meta/1588230740 2024-11-16T12:51:39,765 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T12:51:39,765 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T12:51:39,765 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T12:51:39,767 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T12:51:39,768 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74394290, jitterRate=0.10856130719184875}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T12:51:39,768 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T12:51:39,769 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731761499755Writing region info on filesystem at 1731761499755Initializing all the Stores at 1731761499756 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761499756Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761499756Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761499756Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761499756Cleaning up temporary data from old regions at 1731761499765 (+9 ms)Running coprocessor post-open hooks at 1731761499768 (+3 ms)Region opened successfully at 1731761499769 (+1 ms) 2024-11-16T12:51:39,770 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731761499731 2024-11-16T12:51:39,772 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T12:51:39,772 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T12:51:39,772 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0450ab8807f5,45113,1731761498684 2024-11-16T12:51:39,773 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0450ab8807f5,45113,1731761498684, state=OPEN 2024-11-16T12:51:39,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x10144fc6ca6001a, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:39,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:39,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x10144fc6ca6001c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:39,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45113-0x10144fc6ca6001b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:39,784 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0450ab8807f5,45113,1731761498684 2024-11-16T12:51:39,784 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:39,784 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:39,784 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:39,784 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:51:39,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-16T12:51:39,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0450ab8807f5,45113,1731761498684 in 206 msec 2024-11-16T12:51:39,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-16T12:51:39,790 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 660 msec 2024-11-16T12:51:39,790 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T12:51:39,790 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-16T12:51:39,792 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T12:51:39,792 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,45113,1731761498684, seqNum=-1] 2024-11-16T12:51:39,792 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T12:51:39,793 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-22-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55295, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T12:51:39,798 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 717 msec 2024-11-16T12:51:39,798 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731761499798, completionTime=-1 2024-11-16T12:51:39,798 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-16T12:51:39,798 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T12:51:39,800 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T12:51:39,800 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731761559800 2024-11-16T12:51:39,800 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731761619800 2024-11-16T12:51:39,800 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec 2024-11-16T12:51:39,800 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,39327,1731761498553-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,800 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,39327,1731761498553-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,800 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,39327,1731761498553-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,800 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0450ab8807f5:39327, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,801 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,801 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,802 DEBUG [master/0450ab8807f5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-16T12:51:39,804 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.001sec 2024-11-16T12:51:39,804 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-16T12:51:39,804 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-16T12:51:39,804 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-16T12:51:39,804 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-16T12:51:39,804 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-16T12:51:39,804 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,39327,1731761498553-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T12:51:39,804 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,39327,1731761498553-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-16T12:51:39,807 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-16T12:51:39,807 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-16T12:51:39,807 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,39327,1731761498553-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,849 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6081c7f0 to 127.0.0.1:53208 with session timeout=1000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@b0a71f6 2024-11-16T12:51:39,862 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c376506, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T12:51:39,862 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T12:51:39,863 DEBUG [ReadOnlyZKClient-127.0.0.1:53208@0x6081c7f0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,45113,1731761498684, seqNum=-1] 2024-11-16T12:51:39,863 DEBUG [ReadOnlyZKClient-127.0.0.1:53208@0x6081c7f0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T12:51:39,865 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-22-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T12:51:39,867 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0450ab8807f5,39327,1731761498553 2024-11-16T12:51:39,880 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0450ab8807f5:0 server-side Connection retries=45 2024-11-16T12:51:39,880 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:39,880 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:39,880 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T12:51:39,880 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:51:39,880 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T12:51:39,880 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T12:51:39,880 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T12:51:39,881 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40215 2024-11-16T12:51:39,882 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40215 connecting to ZooKeeper ensemble=127.0.0.1:53208 2024-11-16T12:51:39,882 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:39,883 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:39,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:402150x0, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:51:39,895 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:402150x0, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-16T12:51:39,895 DEBUG [pool-934-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-16T12:51:39,895 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40215-0x10144fc6ca6001e connected 2024-11-16T12:51:39,895 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T12:51:39,896 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T12:51:39,896 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:40215-0x10144fc6ca6001e, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T12:51:39,898 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40215-0x10144fc6ca6001e, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T12:51:39,898 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40215 2024-11-16T12:51:39,902 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40215 2024-11-16T12:51:39,903 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40215 2024-11-16T12:51:39,903 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40215 2024-11-16T12:51:39,904 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40215 2024-11-16T12:51:39,905 INFO [RS:2;0450ab8807f5:40215 {}] regionserver.HRegionServer(746): ClusterId : 143d46ab-afa8-4557-97d2-293a8ded4462 2024-11-16T12:51:39,905 DEBUG [RS:2;0450ab8807f5:40215 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T12:51:39,912 DEBUG [RS:2;0450ab8807f5:40215 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T12:51:39,912 DEBUG [RS:2;0450ab8807f5:40215 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T12:51:39,920 DEBUG [RS:2;0450ab8807f5:40215 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T12:51:39,920 DEBUG [RS:2;0450ab8807f5:40215 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38f38d47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0450ab8807f5/172.17.0.2:0 2024-11-16T12:51:39,932 DEBUG [RS:2;0450ab8807f5:40215 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;0450ab8807f5:40215 2024-11-16T12:51:39,932 INFO [RS:2;0450ab8807f5:40215 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T12:51:39,932 INFO [RS:2;0450ab8807f5:40215 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T12:51:39,932 DEBUG [RS:2;0450ab8807f5:40215 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T12:51:39,933 INFO [RS:2;0450ab8807f5:40215 {}] regionserver.HRegionServer(2659): reportForDuty to master=0450ab8807f5,39327,1731761498553 with port=40215, startcode=1731761499880 2024-11-16T12:51:39,933 DEBUG [RS:2;0450ab8807f5:40215 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T12:51:39,935 INFO [HMaster-EventLoopGroup-20-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38075, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.11 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T12:51:39,935 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39327 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0450ab8807f5,40215,1731761499880 2024-11-16T12:51:39,935 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39327 {}] master.ServerManager(517): Registering regionserver=0450ab8807f5,40215,1731761499880 2024-11-16T12:51:39,936 DEBUG [RS:2;0450ab8807f5:40215 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:39,936 DEBUG [RS:2;0450ab8807f5:40215 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44975 2024-11-16T12:51:39,936 DEBUG [RS:2;0450ab8807f5:40215 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T12:51:39,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T12:51:39,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x10144fc6ca6001a, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T12:51:39,945 DEBUG [RS:2;0450ab8807f5:40215 {}] zookeeper.ZKUtil(111): regionserver:40215-0x10144fc6ca6001e, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0450ab8807f5,40215,1731761499880 2024-11-16T12:51:39,945 WARN [RS:2;0450ab8807f5:40215 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T12:51:39,945 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0450ab8807f5,40215,1731761499880] 2024-11-16T12:51:39,945 INFO [RS:2;0450ab8807f5:40215 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:51:39,945 DEBUG [RS:2;0450ab8807f5:40215 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,40215,1731761499880 2024-11-16T12:51:39,948 INFO [RS:2;0450ab8807f5:40215 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T12:51:39,949 INFO [RS:2;0450ab8807f5:40215 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T12:51:39,950 INFO [RS:2;0450ab8807f5:40215 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T12:51:39,950 INFO [RS:2;0450ab8807f5:40215 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,950 INFO [RS:2;0450ab8807f5:40215 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T12:51:39,951 INFO [RS:2;0450ab8807f5:40215 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T12:51:39,951 INFO [RS:2;0450ab8807f5:40215 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,951 DEBUG [RS:2;0450ab8807f5:40215 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,951 DEBUG [RS:2;0450ab8807f5:40215 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,951 DEBUG [RS:2;0450ab8807f5:40215 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,951 DEBUG [RS:2;0450ab8807f5:40215 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,951 DEBUG [RS:2;0450ab8807f5:40215 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,951 DEBUG [RS:2;0450ab8807f5:40215 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0, corePoolSize=2, maxPoolSize=2 2024-11-16T12:51:39,951 DEBUG [RS:2;0450ab8807f5:40215 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,951 DEBUG [RS:2;0450ab8807f5:40215 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,951 DEBUG [RS:2;0450ab8807f5:40215 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,951 DEBUG [RS:2;0450ab8807f5:40215 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,951 DEBUG [RS:2;0450ab8807f5:40215 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,951 DEBUG [RS:2;0450ab8807f5:40215 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:51:39,951 DEBUG [RS:2;0450ab8807f5:40215 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:51:39,951 DEBUG [RS:2;0450ab8807f5:40215 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:51:39,952 INFO [RS:2;0450ab8807f5:40215 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,952 INFO [RS:2;0450ab8807f5:40215 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,952 INFO [RS:2;0450ab8807f5:40215 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,952 INFO [RS:2;0450ab8807f5:40215 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,952 INFO [RS:2;0450ab8807f5:40215 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,952 INFO [RS:2;0450ab8807f5:40215 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,40215,1731761499880-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T12:51:39,965 INFO [RS:2;0450ab8807f5:40215 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T12:51:39,965 INFO [RS:2;0450ab8807f5:40215 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,40215,1731761499880-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,965 INFO [RS:2;0450ab8807f5:40215 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,965 INFO [RS:2;0450ab8807f5:40215 {}] regionserver.Replication(171): 0450ab8807f5,40215,1731761499880 started 2024-11-16T12:51:39,977 INFO [RS:2;0450ab8807f5:40215 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:51:39,977 INFO [RS:2;0450ab8807f5:40215 {}] regionserver.HRegionServer(1482): Serving as 0450ab8807f5,40215,1731761499880, RpcServer on 0450ab8807f5/172.17.0.2:40215, sessionid=0x10144fc6ca6001e 2024-11-16T12:51:39,977 DEBUG [RS:2;0450ab8807f5:40215 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T12:51:39,977 DEBUG [RS:2;0450ab8807f5:40215 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0450ab8807f5,40215,1731761499880 2024-11-16T12:51:39,977 DEBUG [RS:2;0450ab8807f5:40215 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,40215,1731761499880' 2024-11-16T12:51:39,977 DEBUG [RS:2;0450ab8807f5:40215 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T12:51:39,977 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T12:51:39,978 DEBUG [RS:2;0450ab8807f5:40215 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T12:51:39,978 DEBUG [ReadOnlyZKClient-127.0.0.1:53208@0x6081c7f0 {}] client.AsyncConnectionImpl(321): The fetched master address is 0450ab8807f5,39327,1731761498553 2024-11-16T12:51:39,978 DEBUG [ReadOnlyZKClient-127.0.0.1:53208@0x6081c7f0 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@441f2c71 2024-11-16T12:51:39,978 DEBUG [RS:2;0450ab8807f5:40215 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T12:51:39,978 DEBUG [RS:2;0450ab8807f5:40215 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T12:51:39,978 DEBUG [RS:2;0450ab8807f5:40215 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0450ab8807f5,40215,1731761499880 2024-11-16T12:51:39,978 DEBUG [RS:2;0450ab8807f5:40215 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,40215,1731761499880' 2024-11-16T12:51:39,978 DEBUG [RS:2;0450ab8807f5:40215 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T12:51:39,978 DEBUG [ReadOnlyZKClient-127.0.0.1:53208@0x6081c7f0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T12:51:39,978 DEBUG [RS:2;0450ab8807f5:40215 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T12:51:39,978 DEBUG [RS:2;0450ab8807f5:40215 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T12:51:39,978 INFO [RS:2;0450ab8807f5:40215 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T12:51:39,978 INFO [RS:2;0450ab8807f5:40215 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T12:51:39,979 INFO [HMaster-EventLoopGroup-20-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T12:51:39,980 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39327 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T12:51:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39327 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry 2024-11-16T12:51:39,982 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T12:51:39,983 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:39,983 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39327 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRegionAssignmentAfterMasterRecoveryDueToZKExpiry" procId is: 4 2024-11-16T12:51:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T12:51:39,984 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T12:51:39,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741921_1099 (size=404) 2024-11-16T12:51:39,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741921_1099 (size=404) 2024-11-16T12:51:39,992 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4c4a6e6d05c0ed8942f754578d258c83, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1731761499980.4c4a6e6d05c0ed8942f754578d258c83.', STARTKEY => '', ENDKEY => 'a'}, tableDescriptor='testRegionAssignmentAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:39,992 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 3975dbb20d3790857ef1af53ef826e83, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1731761499980.3975dbb20d3790857ef1af53ef826e83.', STARTKEY => 'b', ENDKEY => 'c'}, tableDescriptor='testRegionAssignmentAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:39,993 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 454d6867041c8b378ebdd3f87fca5f11, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1731761499980.454d6867041c8b378ebdd3f87fca5f11.', STARTKEY => 'a', ENDKEY => 'b'}, tableDescriptor='testRegionAssignmentAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:40,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741922_1100 (size=89) 2024-11-16T12:51:40,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741923_1101 (size=89) 2024-11-16T12:51:40,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741924_1102 (size=88) 2024-11-16T12:51:40,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741922_1100 (size=89) 2024-11-16T12:51:40,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741923_1101 (size=89) 2024-11-16T12:51:40,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741924_1102 (size=88) 2024-11-16T12:51:40,004 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1731761499980.3975dbb20d3790857ef1af53ef826e83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:40,004 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1722): Closing 3975dbb20d3790857ef1af53ef826e83, disabling compactions & flushes 2024-11-16T12:51:40,004 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1731761499980.3975dbb20d3790857ef1af53ef826e83. 2024-11-16T12:51:40,004 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1731761499980.3975dbb20d3790857ef1af53ef826e83. 2024-11-16T12:51:40,004 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1731761499980.3975dbb20d3790857ef1af53ef826e83. after waiting 0 ms 2024-11-16T12:51:40,004 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1731761499980.3975dbb20d3790857ef1af53ef826e83. 2024-11-16T12:51:40,004 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1731761499980.3975dbb20d3790857ef1af53ef826e83. 2024-11-16T12:51:40,004 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1676): Region close journal for 3975dbb20d3790857ef1af53ef826e83: Waiting for close lock at 1731761500004Disabling compacts and flushes for region at 1731761500004Disabling writes for close at 1731761500004Writing region close event to WAL at 1731761500004Closed at 1731761500004 2024-11-16T12:51:40,004 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => fed5462f2af6a59954ccad57d1eb36f1, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1731761499980.fed5462f2af6a59954ccad57d1eb36f1.', STARTKEY => 'c', ENDKEY => 'd'}, tableDescriptor='testRegionAssignmentAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:40,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741925_1103 (size=89) 2024-11-16T12:51:40,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741925_1103 (size=89) 2024-11-16T12:51:40,079 INFO [RS:2;0450ab8807f5:40215 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T12:51:40,081 INFO [RS:2;0450ab8807f5:40215 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0450ab8807f5%2C40215%2C1731761499880, suffix=, logDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,40215,1731761499880, archiveDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/oldWALs, maxLogs=32 2024-11-16T12:51:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T12:51:40,092 DEBUG [RS:2;0450ab8807f5:40215 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,40215,1731761499880/0450ab8807f5%2C40215%2C1731761499880.1731761500081, exclude list is [], retry=0 2024-11-16T12:51:40,095 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36961,DS-7ef9e9a6-250e-467c-989b-a9ffcab2299e,DISK] 2024-11-16T12:51:40,095 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43257,DS-06d91055-02a4-40d7-bddb-15aee81f02ac,DISK] 2024-11-16T12:51:40,097 INFO [RS:2;0450ab8807f5:40215 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,40215,1731761499880/0450ab8807f5%2C40215%2C1731761499880.1731761500081 2024-11-16T12:51:40,097 DEBUG [RS:2;0450ab8807f5:40215 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46667:46667),(127.0.0.1/127.0.0.1:38819:38819)] 2024-11-16T12:51:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T12:51:40,404 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1731761499980.454d6867041c8b378ebdd3f87fca5f11.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:40,404 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1722): Closing 454d6867041c8b378ebdd3f87fca5f11, disabling compactions & flushes 2024-11-16T12:51:40,404 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1731761499980.4c4a6e6d05c0ed8942f754578d258c83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:40,404 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1731761499980.454d6867041c8b378ebdd3f87fca5f11. 2024-11-16T12:51:40,404 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1731761499980.454d6867041c8b378ebdd3f87fca5f11. 2024-11-16T12:51:40,404 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1722): Closing 4c4a6e6d05c0ed8942f754578d258c83, disabling compactions & flushes 2024-11-16T12:51:40,404 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1731761499980.4c4a6e6d05c0ed8942f754578d258c83. 2024-11-16T12:51:40,404 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1731761499980.454d6867041c8b378ebdd3f87fca5f11. after waiting 0 ms 2024-11-16T12:51:40,404 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1731761499980.4c4a6e6d05c0ed8942f754578d258c83. 2024-11-16T12:51:40,404 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1731761499980.454d6867041c8b378ebdd3f87fca5f11. 2024-11-16T12:51:40,404 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1731761499980.4c4a6e6d05c0ed8942f754578d258c83. after waiting 0 ms 2024-11-16T12:51:40,404 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1731761499980.4c4a6e6d05c0ed8942f754578d258c83. 2024-11-16T12:51:40,404 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1731761499980.454d6867041c8b378ebdd3f87fca5f11. 2024-11-16T12:51:40,404 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1731761499980.4c4a6e6d05c0ed8942f754578d258c83. 2024-11-16T12:51:40,404 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4c4a6e6d05c0ed8942f754578d258c83: Waiting for close lock at 1731761500404Disabling compacts and flushes for region at 1731761500404Disabling writes for close at 1731761500404Writing region close event to WAL at 1731761500404Closed at 1731761500404 2024-11-16T12:51:40,404 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1676): Region close journal for 454d6867041c8b378ebdd3f87fca5f11: Waiting for close lock at 1731761500404Disabling compacts and flushes for region at 1731761500404Disabling writes for close at 1731761500404Writing region close event to WAL at 1731761500404Closed at 1731761500404 2024-11-16T12:51:40,405 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1e95ff5fb519b8c6e236266576f123a3, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1731761499980.1e95ff5fb519b8c6e236266576f123a3.', STARTKEY => 'e', ENDKEY => 'f'}, tableDescriptor='testRegionAssignmentAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:40,406 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => ca39b850f78d2f0a8e65d12ae3aff16c, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1731761499980.ca39b850f78d2f0a8e65d12ae3aff16c.', STARTKEY => 'd', ENDKEY => 'e'}, tableDescriptor='testRegionAssignmentAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:40,412 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1731761499980.fed5462f2af6a59954ccad57d1eb36f1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:40,412 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1722): Closing fed5462f2af6a59954ccad57d1eb36f1, disabling compactions & flushes 2024-11-16T12:51:40,412 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1731761499980.fed5462f2af6a59954ccad57d1eb36f1. 2024-11-16T12:51:40,412 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1731761499980.fed5462f2af6a59954ccad57d1eb36f1. 2024-11-16T12:51:40,412 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1731761499980.fed5462f2af6a59954ccad57d1eb36f1. after waiting 0 ms 2024-11-16T12:51:40,412 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1731761499980.fed5462f2af6a59954ccad57d1eb36f1. 2024-11-16T12:51:40,412 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1731761499980.fed5462f2af6a59954ccad57d1eb36f1. 2024-11-16T12:51:40,412 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1676): Region close journal for fed5462f2af6a59954ccad57d1eb36f1: Waiting for close lock at 1731761500412Disabling compacts and flushes for region at 1731761500412Disabling writes for close at 1731761500412Writing region close event to WAL at 1731761500412Closed at 1731761500412 2024-11-16T12:51:40,413 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 40aa1e14f3082cbed8ab6e2e88ab9432, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1731761499980.40aa1e14f3082cbed8ab6e2e88ab9432.', STARTKEY => 'f', ENDKEY => 'g'}, tableDescriptor='testRegionAssignmentAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:40,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741927_1105 (size=89) 2024-11-16T12:51:40,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741927_1105 (size=89) 2024-11-16T12:51:40,414 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1731761499980.1e95ff5fb519b8c6e236266576f123a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:40,414 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1722): Closing 1e95ff5fb519b8c6e236266576f123a3, disabling compactions & flushes 2024-11-16T12:51:40,414 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1731761499980.1e95ff5fb519b8c6e236266576f123a3. 2024-11-16T12:51:40,414 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1731761499980.1e95ff5fb519b8c6e236266576f123a3. 2024-11-16T12:51:40,414 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1731761499980.1e95ff5fb519b8c6e236266576f123a3. after waiting 0 ms 2024-11-16T12:51:40,414 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1731761499980.1e95ff5fb519b8c6e236266576f123a3. 2024-11-16T12:51:40,414 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1731761499980.1e95ff5fb519b8c6e236266576f123a3. 2024-11-16T12:51:40,414 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1e95ff5fb519b8c6e236266576f123a3: Waiting for close lock at 1731761500414Disabling compacts and flushes for region at 1731761500414Disabling writes for close at 1731761500414Writing region close event to WAL at 1731761500414Closed at 1731761500414 2024-11-16T12:51:40,415 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4d811d6d28d7d1423e02635376541525, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1731761499980.4d811d6d28d7d1423e02635376541525.', STARTKEY => 'g', ENDKEY => 'h'}, tableDescriptor='testRegionAssignmentAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:40,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741928_1106 (size=89) 2024-11-16T12:51:40,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741928_1106 (size=89) 2024-11-16T12:51:40,418 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1731761499980.ca39b850f78d2f0a8e65d12ae3aff16c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:40,418 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1722): Closing ca39b850f78d2f0a8e65d12ae3aff16c, disabling compactions & flushes 2024-11-16T12:51:40,418 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1731761499980.ca39b850f78d2f0a8e65d12ae3aff16c. 2024-11-16T12:51:40,418 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1731761499980.ca39b850f78d2f0a8e65d12ae3aff16c. 2024-11-16T12:51:40,418 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1731761499980.ca39b850f78d2f0a8e65d12ae3aff16c. after waiting 0 ms 2024-11-16T12:51:40,418 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1731761499980.ca39b850f78d2f0a8e65d12ae3aff16c. 2024-11-16T12:51:40,418 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1731761499980.ca39b850f78d2f0a8e65d12ae3aff16c. 2024-11-16T12:51:40,418 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1676): Region close journal for ca39b850f78d2f0a8e65d12ae3aff16c: Waiting for close lock at 1731761500418Disabling compacts and flushes for region at 1731761500418Disabling writes for close at 1731761500418Writing region close event to WAL at 1731761500418Closed at 1731761500418 2024-11-16T12:51:40,419 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => cb28d34770f7e0c8ce5cfa1e343a5743, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1731761499980.cb28d34770f7e0c8ce5cfa1e343a5743.', STARTKEY => 'h', ENDKEY => 'i'}, tableDescriptor='testRegionAssignmentAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:40,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741929_1107 (size=89) 2024-11-16T12:51:40,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741929_1107 (size=89) 2024-11-16T12:51:40,423 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1731761499980.40aa1e14f3082cbed8ab6e2e88ab9432.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:40,423 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1722): Closing 40aa1e14f3082cbed8ab6e2e88ab9432, disabling compactions & flushes 2024-11-16T12:51:40,423 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1731761499980.40aa1e14f3082cbed8ab6e2e88ab9432. 2024-11-16T12:51:40,423 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1731761499980.40aa1e14f3082cbed8ab6e2e88ab9432. 2024-11-16T12:51:40,423 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1731761499980.40aa1e14f3082cbed8ab6e2e88ab9432. after waiting 0 ms 2024-11-16T12:51:40,423 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1731761499980.40aa1e14f3082cbed8ab6e2e88ab9432. 2024-11-16T12:51:40,423 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1731761499980.40aa1e14f3082cbed8ab6e2e88ab9432. 2024-11-16T12:51:40,424 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1676): Region close journal for 40aa1e14f3082cbed8ab6e2e88ab9432: Waiting for close lock at 1731761500423Disabling compacts and flushes for region at 1731761500423Disabling writes for close at 1731761500423Writing region close event to WAL at 1731761500423Closed at 1731761500423 2024-11-16T12:51:40,424 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => b7ddacc6e0bab141a9de585b8ff7993a, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1731761499980.b7ddacc6e0bab141a9de585b8ff7993a.', STARTKEY => 'i', ENDKEY => 'j'}, tableDescriptor='testRegionAssignmentAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:40,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741930_1108 (size=89) 2024-11-16T12:51:40,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741930_1108 (size=89) 2024-11-16T12:51:40,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741931_1109 (size=89) 2024-11-16T12:51:40,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741931_1109 (size=89) 2024-11-16T12:51:40,433 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1731761499980.4d811d6d28d7d1423e02635376541525.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:40,434 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1722): Closing 4d811d6d28d7d1423e02635376541525, disabling compactions & flushes 2024-11-16T12:51:40,434 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1731761499980.4d811d6d28d7d1423e02635376541525. 2024-11-16T12:51:40,434 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1731761499980.4d811d6d28d7d1423e02635376541525. 2024-11-16T12:51:40,434 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1731761499980.4d811d6d28d7d1423e02635376541525. after waiting 0 ms 2024-11-16T12:51:40,434 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1731761499980.4d811d6d28d7d1423e02635376541525. 2024-11-16T12:51:40,434 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1731761499980.4d811d6d28d7d1423e02635376541525. 2024-11-16T12:51:40,434 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4d811d6d28d7d1423e02635376541525: Waiting for close lock at 1731761500434Disabling compacts and flushes for region at 1731761500434Disabling writes for close at 1731761500434Writing region close event to WAL at 1731761500434Closed at 1731761500434 2024-11-16T12:51:40,434 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d72159177cd75fa309be8f198cc5236c, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1731761499980.d72159177cd75fa309be8f198cc5236c.', STARTKEY => 'j', ENDKEY => ''}, tableDescriptor='testRegionAssignmentAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b 2024-11-16T12:51:40,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741932_1110 (size=89) 2024-11-16T12:51:40,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741932_1110 (size=89) 2024-11-16T12:51:40,436 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1731761499980.b7ddacc6e0bab141a9de585b8ff7993a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:40,436 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1722): Closing b7ddacc6e0bab141a9de585b8ff7993a, disabling compactions & flushes 2024-11-16T12:51:40,436 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1731761499980.b7ddacc6e0bab141a9de585b8ff7993a. 2024-11-16T12:51:40,436 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1731761499980.b7ddacc6e0bab141a9de585b8ff7993a. 2024-11-16T12:51:40,436 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1731761499980.b7ddacc6e0bab141a9de585b8ff7993a. after waiting 0 ms 2024-11-16T12:51:40,436 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1731761499980.b7ddacc6e0bab141a9de585b8ff7993a. 2024-11-16T12:51:40,436 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1731761499980.b7ddacc6e0bab141a9de585b8ff7993a. 2024-11-16T12:51:40,436 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1676): Region close journal for b7ddacc6e0bab141a9de585b8ff7993a: Waiting for close lock at 1731761500436Disabling compacts and flushes for region at 1731761500436Disabling writes for close at 1731761500436Writing region close event to WAL at 1731761500436Closed at 1731761500436 2024-11-16T12:51:40,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741933_1111 (size=88) 2024-11-16T12:51:40,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741933_1111 (size=88) 2024-11-16T12:51:40,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T12:51:40,834 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1731761499980.cb28d34770f7e0c8ce5cfa1e343a5743.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:40,834 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1722): Closing cb28d34770f7e0c8ce5cfa1e343a5743, disabling compactions & flushes 2024-11-16T12:51:40,834 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1731761499980.cb28d34770f7e0c8ce5cfa1e343a5743. 2024-11-16T12:51:40,835 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1731761499980.cb28d34770f7e0c8ce5cfa1e343a5743. 2024-11-16T12:51:40,835 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1731761499980.cb28d34770f7e0c8ce5cfa1e343a5743. after waiting 0 ms 2024-11-16T12:51:40,835 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1731761499980.cb28d34770f7e0c8ce5cfa1e343a5743. 2024-11-16T12:51:40,835 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1731761499980.cb28d34770f7e0c8ce5cfa1e343a5743. 2024-11-16T12:51:40,835 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1676): Region close journal for cb28d34770f7e0c8ce5cfa1e343a5743: Waiting for close lock at 1731761500834Disabling compacts and flushes for region at 1731761500834Disabling writes for close at 1731761500835 (+1 ms)Writing region close event to WAL at 1731761500835Closed at 1731761500835 2024-11-16T12:51:40,842 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1731761499980.d72159177cd75fa309be8f198cc5236c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:40,842 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1722): Closing d72159177cd75fa309be8f198cc5236c, disabling compactions & flushes 2024-11-16T12:51:40,842 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1731761499980.d72159177cd75fa309be8f198cc5236c. 2024-11-16T12:51:40,842 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1731761499980.d72159177cd75fa309be8f198cc5236c. 2024-11-16T12:51:40,842 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1731761499980.d72159177cd75fa309be8f198cc5236c. after waiting 0 ms 2024-11-16T12:51:40,842 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1731761499980.d72159177cd75fa309be8f198cc5236c. 2024-11-16T12:51:40,842 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1731761499980.d72159177cd75fa309be8f198cc5236c. 2024-11-16T12:51:40,842 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1676): Region close journal for d72159177cd75fa309be8f198cc5236c: Waiting for close lock at 1731761500842Disabling compacts and flushes for region at 1731761500842Disabling writes for close at 1731761500842Writing region close event to WAL at 1731761500842Closed at 1731761500842 2024-11-16T12:51:40,843 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T12:51:40,844 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1731761499980.3975dbb20d3790857ef1af53ef826e83.","families":{"info":[{"qualifier":"regioninfo","vlen":88,"tag":[],"timestamp":"1731761500843"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731761500843"}]},"ts":"1731761500843"} 2024-11-16T12:51:40,844 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1731761499980.4c4a6e6d05c0ed8942f754578d258c83.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1731761500843"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731761500843"}]},"ts":"1731761500843"} 2024-11-16T12:51:40,844 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1731761499980.454d6867041c8b378ebdd3f87fca5f11.","families":{"info":[{"qualifier":"regioninfo","vlen":88,"tag":[],"timestamp":"1731761500843"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731761500843"}]},"ts":"1731761500843"} 2024-11-16T12:51:40,844 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1731761499980.fed5462f2af6a59954ccad57d1eb36f1.","families":{"info":[{"qualifier":"regioninfo","vlen":88,"tag":[],"timestamp":"1731761500843"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731761500843"}]},"ts":"1731761500843"} 2024-11-16T12:51:40,844 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1731761499980.1e95ff5fb519b8c6e236266576f123a3.","families":{"info":[{"qualifier":"regioninfo","vlen":88,"tag":[],"timestamp":"1731761500843"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731761500843"}]},"ts":"1731761500843"} 2024-11-16T12:51:40,845 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1731761499980.ca39b850f78d2f0a8e65d12ae3aff16c.","families":{"info":[{"qualifier":"regioninfo","vlen":88,"tag":[],"timestamp":"1731761500843"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731761500843"}]},"ts":"1731761500843"} 2024-11-16T12:51:40,845 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1731761499980.40aa1e14f3082cbed8ab6e2e88ab9432.","families":{"info":[{"qualifier":"regioninfo","vlen":88,"tag":[],"timestamp":"1731761500843"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731761500843"}]},"ts":"1731761500843"} 2024-11-16T12:51:40,845 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1731761499980.4d811d6d28d7d1423e02635376541525.","families":{"info":[{"qualifier":"regioninfo","vlen":88,"tag":[],"timestamp":"1731761500843"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731761500843"}]},"ts":"1731761500843"} 2024-11-16T12:51:40,845 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1731761499980.b7ddacc6e0bab141a9de585b8ff7993a.","families":{"info":[{"qualifier":"regioninfo","vlen":88,"tag":[],"timestamp":"1731761500843"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731761500843"}]},"ts":"1731761500843"} 2024-11-16T12:51:40,845 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1731761499980.cb28d34770f7e0c8ce5cfa1e343a5743.","families":{"info":[{"qualifier":"regioninfo","vlen":88,"tag":[],"timestamp":"1731761500843"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731761500843"}]},"ts":"1731761500843"} 2024-11-16T12:51:40,845 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1731761499980.d72159177cd75fa309be8f198cc5236c.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1731761500843"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731761500843"}]},"ts":"1731761500843"} 2024-11-16T12:51:40,851 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 11 regions to meta. 2024-11-16T12:51:40,852 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T12:51:40,853 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRegionAssignmentAfterMasterRecoveryDueToZKExpiry","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731761500853"}]},"ts":"1731761500853"} 2024-11-16T12:51:40,855 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, state=ENABLING in hbase:meta 2024-11-16T12:51:40,855 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {0450ab8807f5=0} racks are {/default-rack=0} 2024-11-16T12:51:40,859 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T12:51:40,859 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T12:51:40,859 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-16T12:51:40,859 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T12:51:40,859 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T12:51:40,859 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-16T12:51:40,859 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T12:51:40,860 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T12:51:40,860 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-16T12:51:40,860 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T12:51:40,860 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=4c4a6e6d05c0ed8942f754578d258c83, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=454d6867041c8b378ebdd3f87fca5f11, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=3975dbb20d3790857ef1af53ef826e83, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=fed5462f2af6a59954ccad57d1eb36f1, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=ca39b850f78d2f0a8e65d12ae3aff16c, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=1e95ff5fb519b8c6e236266576f123a3, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=40aa1e14f3082cbed8ab6e2e88ab9432, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=4d811d6d28d7d1423e02635376541525, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=cb28d34770f7e0c8ce5cfa1e343a5743, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=b7ddacc6e0bab141a9de585b8ff7993a, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=d72159177cd75fa309be8f198cc5236c, ASSIGN}] 2024-11-16T12:51:40,862 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=454d6867041c8b378ebdd3f87fca5f11, ASSIGN 2024-11-16T12:51:40,862 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=3975dbb20d3790857ef1af53ef826e83, ASSIGN 2024-11-16T12:51:40,862 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=fed5462f2af6a59954ccad57d1eb36f1, ASSIGN 2024-11-16T12:51:40,862 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=4c4a6e6d05c0ed8942f754578d258c83, ASSIGN 2024-11-16T12:51:40,863 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=454d6867041c8b378ebdd3f87fca5f11, ASSIGN; state=OFFLINE, location=0450ab8807f5,46757,1731761498722; forceNewPlan=false, retain=false 2024-11-16T12:51:40,863 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=fed5462f2af6a59954ccad57d1eb36f1, ASSIGN; state=OFFLINE, location=0450ab8807f5,40215,1731761499880; forceNewPlan=false, retain=false 2024-11-16T12:51:40,863 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=3975dbb20d3790857ef1af53ef826e83, ASSIGN; state=OFFLINE, location=0450ab8807f5,40215,1731761499880; forceNewPlan=false, retain=false 2024-11-16T12:51:40,863 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=4c4a6e6d05c0ed8942f754578d258c83, ASSIGN; state=OFFLINE, location=0450ab8807f5,45113,1731761498684; forceNewPlan=false, retain=false 2024-11-16T12:51:40,866 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=b7ddacc6e0bab141a9de585b8ff7993a, ASSIGN 2024-11-16T12:51:40,866 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=cb28d34770f7e0c8ce5cfa1e343a5743, ASSIGN 2024-11-16T12:51:40,866 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=4d811d6d28d7d1423e02635376541525, ASSIGN 2024-11-16T12:51:40,866 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=d72159177cd75fa309be8f198cc5236c, ASSIGN 2024-11-16T12:51:40,867 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=40aa1e14f3082cbed8ab6e2e88ab9432, ASSIGN 2024-11-16T12:51:40,867 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=cb28d34770f7e0c8ce5cfa1e343a5743, ASSIGN; state=OFFLINE, location=0450ab8807f5,40215,1731761499880; forceNewPlan=false, retain=false 2024-11-16T12:51:40,867 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=4d811d6d28d7d1423e02635376541525, ASSIGN; state=OFFLINE, location=0450ab8807f5,45113,1731761498684; forceNewPlan=false, retain=false 2024-11-16T12:51:40,867 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=b7ddacc6e0bab141a9de585b8ff7993a, ASSIGN; state=OFFLINE, location=0450ab8807f5,46757,1731761498722; forceNewPlan=false, retain=false 2024-11-16T12:51:40,867 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=d72159177cd75fa309be8f198cc5236c, ASSIGN; state=OFFLINE, location=0450ab8807f5,45113,1731761498684; forceNewPlan=false, retain=false 2024-11-16T12:51:40,868 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=40aa1e14f3082cbed8ab6e2e88ab9432, ASSIGN; state=OFFLINE, location=0450ab8807f5,40215,1731761499880; forceNewPlan=false, retain=false 2024-11-16T12:51:40,871 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=ca39b850f78d2f0a8e65d12ae3aff16c, ASSIGN 2024-11-16T12:51:40,871 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=1e95ff5fb519b8c6e236266576f123a3, ASSIGN 2024-11-16T12:51:40,872 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=ca39b850f78d2f0a8e65d12ae3aff16c, ASSIGN; state=OFFLINE, location=0450ab8807f5,46757,1731761498722; forceNewPlan=false, retain=false 2024-11-16T12:51:40,872 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=1e95ff5fb519b8c6e236266576f123a3, ASSIGN; state=OFFLINE, location=0450ab8807f5,45113,1731761498684; forceNewPlan=false, retain=false 2024-11-16T12:51:41,014 INFO [0450ab8807f5:39327 {}] balancer.BaseLoadBalancer(388): Reassigned 11 regions. 11 retained the pre-restart assignment. 2024-11-16T12:51:41,014 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=d72159177cd75fa309be8f198cc5236c, regionState=OPENING, regionLocation=0450ab8807f5,45113,1731761498684 2024-11-16T12:51:41,014 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=4d811d6d28d7d1423e02635376541525, regionState=OPENING, regionLocation=0450ab8807f5,45113,1731761498684 2024-11-16T12:51:41,014 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=b7ddacc6e0bab141a9de585b8ff7993a, regionState=OPENING, regionLocation=0450ab8807f5,46757,1731761498722 2024-11-16T12:51:41,014 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=454d6867041c8b378ebdd3f87fca5f11, regionState=OPENING, regionLocation=0450ab8807f5,46757,1731761498722 2024-11-16T12:51:41,014 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=ca39b850f78d2f0a8e65d12ae3aff16c, regionState=OPENING, regionLocation=0450ab8807f5,46757,1731761498722 2024-11-16T12:51:41,015 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=1e95ff5fb519b8c6e236266576f123a3, regionState=OPENING, regionLocation=0450ab8807f5,45113,1731761498684 2024-11-16T12:51:41,015 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4c4a6e6d05c0ed8942f754578d258c83, regionState=OPENING, regionLocation=0450ab8807f5,45113,1731761498684 2024-11-16T12:51:41,015 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=cb28d34770f7e0c8ce5cfa1e343a5743, regionState=OPENING, regionLocation=0450ab8807f5,40215,1731761499880 2024-11-16T12:51:41,015 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=40aa1e14f3082cbed8ab6e2e88ab9432, regionState=OPENING, regionLocation=0450ab8807f5,40215,1731761499880 2024-11-16T12:51:41,015 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=fed5462f2af6a59954ccad57d1eb36f1, regionState=OPENING, regionLocation=0450ab8807f5,40215,1731761499880 2024-11-16T12:51:41,015 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3975dbb20d3790857ef1af53ef826e83, regionState=OPENING, regionLocation=0450ab8807f5,40215,1731761499880 2024-11-16T12:51:41,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=b7ddacc6e0bab141a9de585b8ff7993a, ASSIGN because future has completed 2024-11-16T12:51:41,017 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7ddacc6e0bab141a9de585b8ff7993a, server=0450ab8807f5,46757,1731761498722}] 2024-11-16T12:51:41,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=ca39b850f78d2f0a8e65d12ae3aff16c, ASSIGN because future has completed 2024-11-16T12:51:41,019 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure ca39b850f78d2f0a8e65d12ae3aff16c, server=0450ab8807f5,46757,1731761498722}] 2024-11-16T12:51:41,019 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=454d6867041c8b378ebdd3f87fca5f11, ASSIGN because future has completed 2024-11-16T12:51:41,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 454d6867041c8b378ebdd3f87fca5f11, server=0450ab8807f5,46757,1731761498722}] 2024-11-16T12:51:41,020 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=d72159177cd75fa309be8f198cc5236c, ASSIGN because future has completed 2024-11-16T12:51:41,021 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=4d811d6d28d7d1423e02635376541525, ASSIGN because future has completed 2024-11-16T12:51:41,022 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=4c4a6e6d05c0ed8942f754578d258c83, ASSIGN because future has completed 2024-11-16T12:51:41,022 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure d72159177cd75fa309be8f198cc5236c, server=0450ab8807f5,45113,1731761498684}] 2024-11-16T12:51:41,022 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d811d6d28d7d1423e02635376541525, server=0450ab8807f5,45113,1731761498684}] 2024-11-16T12:51:41,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=1e95ff5fb519b8c6e236266576f123a3, ASSIGN because future has completed 2024-11-16T12:51:41,026 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=40aa1e14f3082cbed8ab6e2e88ab9432, ASSIGN because future has completed 2024-11-16T12:51:41,027 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=fed5462f2af6a59954ccad57d1eb36f1, ASSIGN because future has completed 2024-11-16T12:51:41,027 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c4a6e6d05c0ed8942f754578d258c83, server=0450ab8807f5,45113,1731761498684}] 2024-11-16T12:51:41,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40aa1e14f3082cbed8ab6e2e88ab9432, server=0450ab8807f5,40215,1731761499880}] 2024-11-16T12:51:41,028 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure fed5462f2af6a59954ccad57d1eb36f1, server=0450ab8807f5,40215,1731761499880}] 2024-11-16T12:51:41,028 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=cb28d34770f7e0c8ce5cfa1e343a5743, ASSIGN because future has completed 2024-11-16T12:51:41,029 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=3975dbb20d3790857ef1af53ef826e83, ASSIGN because future has completed 2024-11-16T12:51:41,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3975dbb20d3790857ef1af53ef826e83, server=0450ab8807f5,40215,1731761499880}] 2024-11-16T12:51:41,033 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e95ff5fb519b8c6e236266576f123a3, server=0450ab8807f5,45113,1731761498684}] 2024-11-16T12:51:41,033 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb28d34770f7e0c8ce5cfa1e343a5743, server=0450ab8807f5,40215,1731761499880}] 2024-11-16T12:51:41,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T12:51:41,170 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T12:51:41,172 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-23-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59769, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T12:51:41,176 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1731761499980.454d6867041c8b378ebdd3f87fca5f11. 2024-11-16T12:51:41,176 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => 454d6867041c8b378ebdd3f87fca5f11, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1731761499980.454d6867041c8b378ebdd3f87fca5f11.', STARTKEY => 'a', ENDKEY => 'b'} 2024-11-16T12:51:41,176 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRegionAssignmentAfterMasterRecoveryDueToZKExpiry 454d6867041c8b378ebdd3f87fca5f11 2024-11-16T12:51:41,177 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1731761499980.454d6867041c8b378ebdd3f87fca5f11.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:41,177 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for 454d6867041c8b378ebdd3f87fca5f11 2024-11-16T12:51:41,177 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for 454d6867041c8b378ebdd3f87fca5f11 2024-11-16T12:51:41,179 INFO [StoreOpener-454d6867041c8b378ebdd3f87fca5f11-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 454d6867041c8b378ebdd3f87fca5f11 2024-11-16T12:51:41,180 INFO [StoreOpener-454d6867041c8b378ebdd3f87fca5f11-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 454d6867041c8b378ebdd3f87fca5f11 columnFamilyName info 2024-11-16T12:51:41,180 DEBUG [StoreOpener-454d6867041c8b378ebdd3f87fca5f11-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:41,181 INFO [StoreOpener-454d6867041c8b378ebdd3f87fca5f11-1 {}] regionserver.HStore(327): Store=454d6867041c8b378ebdd3f87fca5f11/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:41,181 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for 454d6867041c8b378ebdd3f87fca5f11 2024-11-16T12:51:41,182 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/454d6867041c8b378ebdd3f87fca5f11 2024-11-16T12:51:41,182 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/454d6867041c8b378ebdd3f87fca5f11 2024-11-16T12:51:41,182 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for 454d6867041c8b378ebdd3f87fca5f11 2024-11-16T12:51:41,182 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for 454d6867041c8b378ebdd3f87fca5f11 2024-11-16T12:51:41,183 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T12:51:41,184 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-24-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34735, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T12:51:41,185 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for 454d6867041c8b378ebdd3f87fca5f11 2024-11-16T12:51:41,186 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(132): Open testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1731761499980.1e95ff5fb519b8c6e236266576f123a3. 2024-11-16T12:51:41,186 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7752): Opening region: {ENCODED => 1e95ff5fb519b8c6e236266576f123a3, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1731761499980.1e95ff5fb519b8c6e236266576f123a3.', STARTKEY => 'e', ENDKEY => 'f'} 2024-11-16T12:51:41,187 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRegionAssignmentAfterMasterRecoveryDueToZKExpiry 1e95ff5fb519b8c6e236266576f123a3 2024-11-16T12:51:41,187 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1731761499980.1e95ff5fb519b8c6e236266576f123a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:41,187 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7794): checking encryption for 1e95ff5fb519b8c6e236266576f123a3 2024-11-16T12:51:41,187 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7797): checking classloading for 1e95ff5fb519b8c6e236266576f123a3 2024-11-16T12:51:41,193 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/454d6867041c8b378ebdd3f87fca5f11/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:51:41,194 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened 454d6867041c8b378ebdd3f87fca5f11; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62053401, jitterRate=-0.07533226907253265}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T12:51:41,194 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 454d6867041c8b378ebdd3f87fca5f11 2024-11-16T12:51:41,195 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for 454d6867041c8b378ebdd3f87fca5f11: Running coprocessor pre-open hook at 1731761501177Writing region info on filesystem at 1731761501177Initializing all the Stores at 1731761501178 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761501178Cleaning up temporary data from old regions at 1731761501182 (+4 ms)Running coprocessor post-open hooks at 1731761501194 (+12 ms)Region opened successfully at 1731761501195 (+1 ms) 2024-11-16T12:51:41,196 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1731761499980.454d6867041c8b378ebdd3f87fca5f11., pid=18, masterSystemTime=1731761501170 2024-11-16T12:51:41,196 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(132): Open testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1731761499980.fed5462f2af6a59954ccad57d1eb36f1. 2024-11-16T12:51:41,196 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7752): Opening region: {ENCODED => fed5462f2af6a59954ccad57d1eb36f1, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1731761499980.fed5462f2af6a59954ccad57d1eb36f1.', STARTKEY => 'c', ENDKEY => 'd'} 2024-11-16T12:51:41,197 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRegionAssignmentAfterMasterRecoveryDueToZKExpiry fed5462f2af6a59954ccad57d1eb36f1 2024-11-16T12:51:41,197 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1731761499980.fed5462f2af6a59954ccad57d1eb36f1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:41,197 INFO [StoreOpener-1e95ff5fb519b8c6e236266576f123a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1e95ff5fb519b8c6e236266576f123a3 2024-11-16T12:51:41,197 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7794): checking encryption for fed5462f2af6a59954ccad57d1eb36f1 2024-11-16T12:51:41,197 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7797): checking classloading for fed5462f2af6a59954ccad57d1eb36f1 2024-11-16T12:51:41,198 INFO [StoreOpener-1e95ff5fb519b8c6e236266576f123a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e95ff5fb519b8c6e236266576f123a3 columnFamilyName info 2024-11-16T12:51:41,199 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1731761499980.454d6867041c8b378ebdd3f87fca5f11. 2024-11-16T12:51:41,199 DEBUG [StoreOpener-1e95ff5fb519b8c6e236266576f123a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:41,199 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1731761499980.454d6867041c8b378ebdd3f87fca5f11. 2024-11-16T12:51:41,199 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1731761499980.ca39b850f78d2f0a8e65d12ae3aff16c. 2024-11-16T12:51:41,199 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => ca39b850f78d2f0a8e65d12ae3aff16c, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1731761499980.ca39b850f78d2f0a8e65d12ae3aff16c.', STARTKEY => 'd', ENDKEY => 'e'} 2024-11-16T12:51:41,199 INFO [StoreOpener-fed5462f2af6a59954ccad57d1eb36f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region fed5462f2af6a59954ccad57d1eb36f1 2024-11-16T12:51:41,199 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRegionAssignmentAfterMasterRecoveryDueToZKExpiry ca39b850f78d2f0a8e65d12ae3aff16c 2024-11-16T12:51:41,199 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1731761499980.ca39b850f78d2f0a8e65d12ae3aff16c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:41,199 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for ca39b850f78d2f0a8e65d12ae3aff16c 2024-11-16T12:51:41,199 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for ca39b850f78d2f0a8e65d12ae3aff16c 2024-11-16T12:51:41,199 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=454d6867041c8b378ebdd3f87fca5f11, regionState=OPEN, openSeqNum=2, regionLocation=0450ab8807f5,46757,1731761498722 2024-11-16T12:51:41,200 INFO [StoreOpener-1e95ff5fb519b8c6e236266576f123a3-1 {}] regionserver.HStore(327): Store=1e95ff5fb519b8c6e236266576f123a3/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:41,201 INFO [StoreOpener-fed5462f2af6a59954ccad57d1eb36f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fed5462f2af6a59954ccad57d1eb36f1 columnFamilyName info 2024-11-16T12:51:41,201 DEBUG [StoreOpener-fed5462f2af6a59954ccad57d1eb36f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:41,201 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1038): replaying wal for 1e95ff5fb519b8c6e236266576f123a3 2024-11-16T12:51:41,201 INFO [StoreOpener-fed5462f2af6a59954ccad57d1eb36f1-1 {}] regionserver.HStore(327): Store=fed5462f2af6a59954ccad57d1eb36f1/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:41,201 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1038): replaying wal for fed5462f2af6a59954ccad57d1eb36f1 2024-11-16T12:51:41,201 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/1e95ff5fb519b8c6e236266576f123a3 2024-11-16T12:51:41,202 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 454d6867041c8b378ebdd3f87fca5f11, server=0450ab8807f5,46757,1731761498722 because future has completed 2024-11-16T12:51:41,202 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/fed5462f2af6a59954ccad57d1eb36f1 2024-11-16T12:51:41,202 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/1e95ff5fb519b8c6e236266576f123a3 2024-11-16T12:51:41,203 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/fed5462f2af6a59954ccad57d1eb36f1 2024-11-16T12:51:41,203 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1048): stopping wal replay for 1e95ff5fb519b8c6e236266576f123a3 2024-11-16T12:51:41,203 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1060): Cleaning up temporary data for 1e95ff5fb519b8c6e236266576f123a3 2024-11-16T12:51:41,203 INFO [StoreOpener-ca39b850f78d2f0a8e65d12ae3aff16c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region ca39b850f78d2f0a8e65d12ae3aff16c 2024-11-16T12:51:41,204 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1048): stopping wal replay for fed5462f2af6a59954ccad57d1eb36f1 2024-11-16T12:51:41,204 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1060): Cleaning up temporary data for fed5462f2af6a59954ccad57d1eb36f1 2024-11-16T12:51:41,204 INFO [StoreOpener-ca39b850f78d2f0a8e65d12ae3aff16c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca39b850f78d2f0a8e65d12ae3aff16c columnFamilyName info 2024-11-16T12:51:41,205 DEBUG [StoreOpener-ca39b850f78d2f0a8e65d12ae3aff16c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:41,205 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1093): writing seq id for 1e95ff5fb519b8c6e236266576f123a3 2024-11-16T12:51:41,205 INFO [StoreOpener-ca39b850f78d2f0a8e65d12ae3aff16c-1 {}] regionserver.HStore(327): Store=ca39b850f78d2f0a8e65d12ae3aff16c/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:41,205 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for ca39b850f78d2f0a8e65d12ae3aff16c 2024-11-16T12:51:41,206 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1093): writing seq id for fed5462f2af6a59954ccad57d1eb36f1 2024-11-16T12:51:41,206 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/ca39b850f78d2f0a8e65d12ae3aff16c 2024-11-16T12:51:41,206 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=6 2024-11-16T12:51:41,206 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 454d6867041c8b378ebdd3f87fca5f11, server=0450ab8807f5,46757,1731761498722 in 183 msec 2024-11-16T12:51:41,207 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/ca39b850f78d2f0a8e65d12ae3aff16c 2024-11-16T12:51:41,207 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for ca39b850f78d2f0a8e65d12ae3aff16c 2024-11-16T12:51:41,207 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for ca39b850f78d2f0a8e65d12ae3aff16c 2024-11-16T12:51:41,209 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/1e95ff5fb519b8c6e236266576f123a3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:51:41,209 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=454d6867041c8b378ebdd3f87fca5f11, ASSIGN in 346 msec 2024-11-16T12:51:41,209 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1114): Opened 1e95ff5fb519b8c6e236266576f123a3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67397863, jitterRate=0.004306420683860779}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T12:51:41,209 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1e95ff5fb519b8c6e236266576f123a3 2024-11-16T12:51:41,209 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1006): Region open journal for 1e95ff5fb519b8c6e236266576f123a3: Running coprocessor pre-open hook at 1731761501187Writing region info on filesystem at 1731761501187Initializing all the Stores at 1731761501188 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761501188Cleaning up temporary data from old regions at 1731761501203 (+15 ms)Running coprocessor post-open hooks at 1731761501209 (+6 ms)Region opened successfully at 1731761501209 2024-11-16T12:51:41,209 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for ca39b850f78d2f0a8e65d12ae3aff16c 2024-11-16T12:51:41,210 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2236): Post open deploy tasks for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1731761499980.1e95ff5fb519b8c6e236266576f123a3., pid=26, masterSystemTime=1731761501183 2024-11-16T12:51:41,211 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/fed5462f2af6a59954ccad57d1eb36f1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:51:41,212 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/ca39b850f78d2f0a8e65d12ae3aff16c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:51:41,212 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1114): Opened fed5462f2af6a59954ccad57d1eb36f1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69007141, jitterRate=0.028286531567573547}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T12:51:41,212 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fed5462f2af6a59954ccad57d1eb36f1 2024-11-16T12:51:41,212 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1006): Region open journal for fed5462f2af6a59954ccad57d1eb36f1: Running coprocessor pre-open hook at 1731761501197Writing region info on filesystem at 1731761501197Initializing all the Stores at 1731761501198 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761501198Cleaning up temporary data from old regions at 1731761501204 (+6 ms)Running coprocessor post-open hooks at 1731761501212 (+8 ms)Region opened successfully at 1731761501212 2024-11-16T12:51:41,212 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened ca39b850f78d2f0a8e65d12ae3aff16c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64587926, jitterRate=-0.03756490349769592}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T12:51:41,212 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ca39b850f78d2f0a8e65d12ae3aff16c 2024-11-16T12:51:41,212 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for ca39b850f78d2f0a8e65d12ae3aff16c: Running coprocessor pre-open hook at 1731761501199Writing region info on filesystem at 1731761501199Initializing all the Stores at 1731761501200 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761501201 (+1 ms)Cleaning up temporary data from old regions at 1731761501207 (+6 ms)Running coprocessor post-open hooks at 1731761501212 (+5 ms)Region opened successfully at 1731761501212 2024-11-16T12:51:41,212 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2266): Finished post open deploy task for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1731761499980.1e95ff5fb519b8c6e236266576f123a3. 2024-11-16T12:51:41,213 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(153): Opened testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1731761499980.1e95ff5fb519b8c6e236266576f123a3. 2024-11-16T12:51:41,213 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1731761499980.4c4a6e6d05c0ed8942f754578d258c83. 2024-11-16T12:51:41,213 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1731761499980.ca39b850f78d2f0a8e65d12ae3aff16c., pid=17, masterSystemTime=1731761501170 2024-11-16T12:51:41,213 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 4c4a6e6d05c0ed8942f754578d258c83, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1731761499980.4c4a6e6d05c0ed8942f754578d258c83.', STARTKEY => '', ENDKEY => 'a'} 2024-11-16T12:51:41,213 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2236): Post open deploy tasks for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1731761499980.fed5462f2af6a59954ccad57d1eb36f1., pid=23, masterSystemTime=1731761501182 2024-11-16T12:51:41,213 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRegionAssignmentAfterMasterRecoveryDueToZKExpiry 4c4a6e6d05c0ed8942f754578d258c83 2024-11-16T12:51:41,213 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=1e95ff5fb519b8c6e236266576f123a3, regionState=OPEN, openSeqNum=2, regionLocation=0450ab8807f5,45113,1731761498684 2024-11-16T12:51:41,214 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1731761499980.4c4a6e6d05c0ed8942f754578d258c83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:41,214 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 4c4a6e6d05c0ed8942f754578d258c83 2024-11-16T12:51:41,214 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 4c4a6e6d05c0ed8942f754578d258c83 2024-11-16T12:51:41,220 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1731761499980.ca39b850f78d2f0a8e65d12ae3aff16c. 2024-11-16T12:51:41,220 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1731761499980.ca39b850f78d2f0a8e65d12ae3aff16c. 2024-11-16T12:51:41,220 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(132): Open testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1731761499980.b7ddacc6e0bab141a9de585b8ff7993a. 2024-11-16T12:51:41,220 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7752): Opening region: {ENCODED => b7ddacc6e0bab141a9de585b8ff7993a, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1731761499980.b7ddacc6e0bab141a9de585b8ff7993a.', STARTKEY => 'i', ENDKEY => 'j'} 2024-11-16T12:51:41,221 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRegionAssignmentAfterMasterRecoveryDueToZKExpiry b7ddacc6e0bab141a9de585b8ff7993a 2024-11-16T12:51:41,221 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1731761499980.b7ddacc6e0bab141a9de585b8ff7993a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:41,221 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7794): checking encryption for b7ddacc6e0bab141a9de585b8ff7993a 2024-11-16T12:51:41,221 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7797): checking classloading for b7ddacc6e0bab141a9de585b8ff7993a 2024-11-16T12:51:41,221 INFO [StoreOpener-4c4a6e6d05c0ed8942f754578d258c83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 4c4a6e6d05c0ed8942f754578d258c83 2024-11-16T12:51:41,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e95ff5fb519b8c6e236266576f123a3, server=0450ab8807f5,45113,1731761498684 because future has completed 2024-11-16T12:51:41,222 INFO [StoreOpener-b7ddacc6e0bab141a9de585b8ff7993a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region b7ddacc6e0bab141a9de585b8ff7993a 2024-11-16T12:51:41,223 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=ca39b850f78d2f0a8e65d12ae3aff16c, regionState=OPEN, openSeqNum=2, regionLocation=0450ab8807f5,46757,1731761498722 2024-11-16T12:51:41,223 INFO [StoreOpener-4c4a6e6d05c0ed8942f754578d258c83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c4a6e6d05c0ed8942f754578d258c83 columnFamilyName info 2024-11-16T12:51:41,223 DEBUG [StoreOpener-4c4a6e6d05c0ed8942f754578d258c83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:41,224 INFO [StoreOpener-4c4a6e6d05c0ed8942f754578d258c83-1 {}] regionserver.HStore(327): Store=4c4a6e6d05c0ed8942f754578d258c83/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:41,224 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 4c4a6e6d05c0ed8942f754578d258c83 2024-11-16T12:51:41,225 INFO [StoreOpener-b7ddacc6e0bab141a9de585b8ff7993a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7ddacc6e0bab141a9de585b8ff7993a columnFamilyName info 2024-11-16T12:51:41,225 DEBUG [StoreOpener-b7ddacc6e0bab141a9de585b8ff7993a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:41,225 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=fed5462f2af6a59954ccad57d1eb36f1, regionState=OPEN, openSeqNum=2, regionLocation=0450ab8807f5,40215,1731761499880 2024-11-16T12:51:41,225 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2266): Finished post open deploy task for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1731761499980.fed5462f2af6a59954ccad57d1eb36f1. 2024-11-16T12:51:41,225 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/4c4a6e6d05c0ed8942f754578d258c83 2024-11-16T12:51:41,225 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(153): Opened testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1731761499980.fed5462f2af6a59954ccad57d1eb36f1. 2024-11-16T12:51:41,226 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] handler.AssignRegionHandler(132): Open testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1731761499980.cb28d34770f7e0c8ce5cfa1e343a5743. 2024-11-16T12:51:41,226 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/4c4a6e6d05c0ed8942f754578d258c83 2024-11-16T12:51:41,226 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.HRegion(7752): Opening region: {ENCODED => cb28d34770f7e0c8ce5cfa1e343a5743, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1731761499980.cb28d34770f7e0c8ce5cfa1e343a5743.', STARTKEY => 'h', ENDKEY => 'i'} 2024-11-16T12:51:41,226 INFO [StoreOpener-b7ddacc6e0bab141a9de585b8ff7993a-1 {}] regionserver.HStore(327): Store=b7ddacc6e0bab141a9de585b8ff7993a/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:41,226 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1038): replaying wal for b7ddacc6e0bab141a9de585b8ff7993a 2024-11-16T12:51:41,226 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRegionAssignmentAfterMasterRecoveryDueToZKExpiry cb28d34770f7e0c8ce5cfa1e343a5743 2024-11-16T12:51:41,226 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 4c4a6e6d05c0ed8942f754578d258c83 2024-11-16T12:51:41,226 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 4c4a6e6d05c0ed8942f754578d258c83 2024-11-16T12:51:41,226 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1731761499980.cb28d34770f7e0c8ce5cfa1e343a5743.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:41,227 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.HRegion(7794): checking encryption for cb28d34770f7e0c8ce5cfa1e343a5743 2024-11-16T12:51:41,227 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.HRegion(7797): checking classloading for cb28d34770f7e0c8ce5cfa1e343a5743 2024-11-16T12:51:41,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure ca39b850f78d2f0a8e65d12ae3aff16c, server=0450ab8807f5,46757,1731761498722 because future has completed 2024-11-16T12:51:41,227 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/b7ddacc6e0bab141a9de585b8ff7993a 2024-11-16T12:51:41,228 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/b7ddacc6e0bab141a9de585b8ff7993a 2024-11-16T12:51:41,228 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure fed5462f2af6a59954ccad57d1eb36f1, server=0450ab8807f5,40215,1731761499880 because future has completed 2024-11-16T12:51:41,229 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1048): stopping wal replay for b7ddacc6e0bab141a9de585b8ff7993a 2024-11-16T12:51:41,229 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1060): Cleaning up temporary data for b7ddacc6e0bab141a9de585b8ff7993a 2024-11-16T12:51:41,230 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 4c4a6e6d05c0ed8942f754578d258c83 2024-11-16T12:51:41,230 INFO [StoreOpener-cb28d34770f7e0c8ce5cfa1e343a5743-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region cb28d34770f7e0c8ce5cfa1e343a5743 2024-11-16T12:51:41,231 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=10 2024-11-16T12:51:41,231 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 1e95ff5fb519b8c6e236266576f123a3, server=0450ab8807f5,45113,1731761498684 in 191 msec 2024-11-16T12:51:41,232 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1093): writing seq id for b7ddacc6e0bab141a9de585b8ff7993a 2024-11-16T12:51:41,232 INFO [StoreOpener-cb28d34770f7e0c8ce5cfa1e343a5743-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb28d34770f7e0c8ce5cfa1e343a5743 columnFamilyName info 2024-11-16T12:51:41,232 DEBUG [StoreOpener-cb28d34770f7e0c8ce5cfa1e343a5743-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:41,233 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/4c4a6e6d05c0ed8942f754578d258c83/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:51:41,233 INFO [StoreOpener-cb28d34770f7e0c8ce5cfa1e343a5743-1 {}] regionserver.HStore(327): Store=cb28d34770f7e0c8ce5cfa1e343a5743/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:41,233 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=9 2024-11-16T12:51:41,233 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 4c4a6e6d05c0ed8942f754578d258c83; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73827237, jitterRate=0.10011155903339386}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T12:51:41,233 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4c4a6e6d05c0ed8942f754578d258c83 2024-11-16T12:51:41,234 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 4c4a6e6d05c0ed8942f754578d258c83: Running coprocessor pre-open hook at 1731761501214Writing region info on filesystem at 1731761501214Initializing all the Stores at 1731761501219 (+5 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761501219Cleaning up temporary data from old regions at 1731761501226 (+7 ms)Running coprocessor post-open hooks at 1731761501234 (+8 ms)Region opened successfully at 1731761501234 2024-11-16T12:51:41,234 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.HRegion(1038): replaying wal for cb28d34770f7e0c8ce5cfa1e343a5743 2024-11-16T12:51:41,234 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=1e95ff5fb519b8c6e236266576f123a3, ASSIGN in 371 msec 2024-11-16T12:51:41,235 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/cb28d34770f7e0c8ce5cfa1e343a5743 2024-11-16T12:51:41,235 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/cb28d34770f7e0c8ce5cfa1e343a5743 2024-11-16T12:51:41,236 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure ca39b850f78d2f0a8e65d12ae3aff16c, server=0450ab8807f5,46757,1731761498722 in 211 msec 2024-11-16T12:51:41,236 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1731761499980.4c4a6e6d05c0ed8942f754578d258c83., pid=21, masterSystemTime=1731761501183 2024-11-16T12:51:41,236 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/b7ddacc6e0bab141a9de585b8ff7993a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:51:41,236 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.HRegion(1048): stopping wal replay for cb28d34770f7e0c8ce5cfa1e343a5743 2024-11-16T12:51:41,236 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.HRegion(1060): Cleaning up temporary data for cb28d34770f7e0c8ce5cfa1e343a5743 2024-11-16T12:51:41,237 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1114): Opened b7ddacc6e0bab141a9de585b8ff7993a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64633714, jitterRate=-0.036882609128952026}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T12:51:41,237 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b7ddacc6e0bab141a9de585b8ff7993a 2024-11-16T12:51:41,238 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=8 2024-11-16T12:51:41,238 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure fed5462f2af6a59954ccad57d1eb36f1, server=0450ab8807f5,40215,1731761499880 in 203 msec 2024-11-16T12:51:41,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=ca39b850f78d2f0a8e65d12ae3aff16c, ASSIGN in 374 msec 2024-11-16T12:51:41,239 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1006): Region open journal for b7ddacc6e0bab141a9de585b8ff7993a: Running coprocessor pre-open hook at 1731761501221Writing region info on filesystem at 1731761501221Initializing all the Stores at 1731761501222 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761501222Cleaning up temporary data from old regions at 1731761501229 (+7 ms)Running coprocessor post-open hooks at 1731761501237 (+8 ms)Region opened successfully at 1731761501238 (+1 ms) 2024-11-16T12:51:41,239 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1731761499980.4c4a6e6d05c0ed8942f754578d258c83. 2024-11-16T12:51:41,239 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1731761499980.4c4a6e6d05c0ed8942f754578d258c83. 2024-11-16T12:51:41,239 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(132): Open testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1731761499980.4d811d6d28d7d1423e02635376541525. 2024-11-16T12:51:41,239 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7752): Opening region: {ENCODED => 4d811d6d28d7d1423e02635376541525, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1731761499980.4d811d6d28d7d1423e02635376541525.', STARTKEY => 'g', ENDKEY => 'h'} 2024-11-16T12:51:41,240 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2236): Post open deploy tasks for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1731761499980.b7ddacc6e0bab141a9de585b8ff7993a., pid=16, masterSystemTime=1731761501170 2024-11-16T12:51:41,240 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRegionAssignmentAfterMasterRecoveryDueToZKExpiry 4d811d6d28d7d1423e02635376541525 2024-11-16T12:51:41,240 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1731761499980.4d811d6d28d7d1423e02635376541525.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:41,240 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7794): checking encryption for 4d811d6d28d7d1423e02635376541525 2024-11-16T12:51:41,240 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4c4a6e6d05c0ed8942f754578d258c83, regionState=OPEN, openSeqNum=2, regionLocation=0450ab8807f5,45113,1731761498684 2024-11-16T12:51:41,240 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7797): checking classloading for 4d811d6d28d7d1423e02635376541525 2024-11-16T12:51:41,240 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.HRegion(1093): writing seq id for cb28d34770f7e0c8ce5cfa1e343a5743 2024-11-16T12:51:41,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c4a6e6d05c0ed8942f754578d258c83, server=0450ab8807f5,45113,1731761498684 because future has completed 2024-11-16T12:51:41,243 INFO [StoreOpener-4d811d6d28d7d1423e02635376541525-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 4d811d6d28d7d1423e02635376541525 2024-11-16T12:51:41,243 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/cb28d34770f7e0c8ce5cfa1e343a5743/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:51:41,243 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=fed5462f2af6a59954ccad57d1eb36f1, ASSIGN in 378 msec 2024-11-16T12:51:41,243 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2266): Finished post open deploy task for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1731761499980.b7ddacc6e0bab141a9de585b8ff7993a. 2024-11-16T12:51:41,244 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(153): Opened testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1731761499980.b7ddacc6e0bab141a9de585b8ff7993a. 2024-11-16T12:51:41,244 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.HRegion(1114): Opened cb28d34770f7e0c8ce5cfa1e343a5743; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60955475, jitterRate=-0.09169264137744904}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T12:51:41,244 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cb28d34770f7e0c8ce5cfa1e343a5743 2024-11-16T12:51:41,244 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.HRegion(1006): Region open journal for cb28d34770f7e0c8ce5cfa1e343a5743: Running coprocessor pre-open hook at 1731761501227Writing region info on filesystem at 1731761501227Initializing all the Stores at 1731761501228 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761501229 (+1 ms)Cleaning up temporary data from old regions at 1731761501236 (+7 ms)Running coprocessor post-open hooks at 1731761501244 (+8 ms)Region opened successfully at 1731761501244 2024-11-16T12:51:41,245 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.HRegionServer(2236): Post open deploy tasks for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1731761499980.cb28d34770f7e0c8ce5cfa1e343a5743., pid=25, masterSystemTime=1731761501182 2024-11-16T12:51:41,245 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=b7ddacc6e0bab141a9de585b8ff7993a, regionState=OPEN, openSeqNum=2, regionLocation=0450ab8807f5,46757,1731761498722 2024-11-16T12:51:41,245 INFO [StoreOpener-4d811d6d28d7d1423e02635376541525-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d811d6d28d7d1423e02635376541525 columnFamilyName info 2024-11-16T12:51:41,245 DEBUG [StoreOpener-4d811d6d28d7d1423e02635376541525-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:41,247 INFO [StoreOpener-4d811d6d28d7d1423e02635376541525-1 {}] regionserver.HStore(327): Store=4d811d6d28d7d1423e02635376541525/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:41,247 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.HRegionServer(2266): Finished post open deploy task for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1731761499980.cb28d34770f7e0c8ce5cfa1e343a5743. 2024-11-16T12:51:41,247 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] handler.AssignRegionHandler(153): Opened testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1731761499980.cb28d34770f7e0c8ce5cfa1e343a5743. 2024-11-16T12:51:41,247 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] handler.AssignRegionHandler(132): Open testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1731761499980.40aa1e14f3082cbed8ab6e2e88ab9432. 2024-11-16T12:51:41,247 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1038): replaying wal for 4d811d6d28d7d1423e02635376541525 2024-11-16T12:51:41,247 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(7752): Opening region: {ENCODED => 40aa1e14f3082cbed8ab6e2e88ab9432, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1731761499980.40aa1e14f3082cbed8ab6e2e88ab9432.', STARTKEY => 'f', ENDKEY => 'g'} 2024-11-16T12:51:41,248 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRegionAssignmentAfterMasterRecoveryDueToZKExpiry 40aa1e14f3082cbed8ab6e2e88ab9432 2024-11-16T12:51:41,248 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1731761499980.40aa1e14f3082cbed8ab6e2e88ab9432.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:41,248 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(7794): checking encryption for 40aa1e14f3082cbed8ab6e2e88ab9432 2024-11-16T12:51:41,248 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(7797): checking classloading for 40aa1e14f3082cbed8ab6e2e88ab9432 2024-11-16T12:51:41,248 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7ddacc6e0bab141a9de585b8ff7993a, server=0450ab8807f5,46757,1731761498722 because future has completed 2024-11-16T12:51:41,249 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=cb28d34770f7e0c8ce5cfa1e343a5743, regionState=OPEN, openSeqNum=2, regionLocation=0450ab8807f5,40215,1731761499880 2024-11-16T12:51:41,249 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=5 2024-11-16T12:51:41,249 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 4c4a6e6d05c0ed8942f754578d258c83, server=0450ab8807f5,45113,1731761498684 in 219 msec 2024-11-16T12:51:41,250 INFO [StoreOpener-40aa1e14f3082cbed8ab6e2e88ab9432-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 40aa1e14f3082cbed8ab6e2e88ab9432 2024-11-16T12:51:41,252 INFO [StoreOpener-40aa1e14f3082cbed8ab6e2e88ab9432-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40aa1e14f3082cbed8ab6e2e88ab9432 columnFamilyName info 2024-11-16T12:51:41,252 DEBUG [StoreOpener-40aa1e14f3082cbed8ab6e2e88ab9432-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:41,252 INFO [StoreOpener-40aa1e14f3082cbed8ab6e2e88ab9432-1 {}] regionserver.HStore(327): Store=40aa1e14f3082cbed8ab6e2e88ab9432/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:41,252 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1038): replaying wal for 40aa1e14f3082cbed8ab6e2e88ab9432 2024-11-16T12:51:41,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb28d34770f7e0c8ce5cfa1e343a5743, server=0450ab8807f5,40215,1731761499880 because future has completed 2024-11-16T12:51:41,254 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/40aa1e14f3082cbed8ab6e2e88ab9432 2024-11-16T12:51:41,254 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/40aa1e14f3082cbed8ab6e2e88ab9432 2024-11-16T12:51:41,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=4c4a6e6d05c0ed8942f754578d258c83, ASSIGN in 389 msec 2024-11-16T12:51:41,254 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/4d811d6d28d7d1423e02635376541525 2024-11-16T12:51:41,255 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/4d811d6d28d7d1423e02635376541525 2024-11-16T12:51:41,255 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1048): stopping wal replay for 40aa1e14f3082cbed8ab6e2e88ab9432 2024-11-16T12:51:41,255 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1060): Cleaning up temporary data for 40aa1e14f3082cbed8ab6e2e88ab9432 2024-11-16T12:51:41,255 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1048): stopping wal replay for 4d811d6d28d7d1423e02635376541525 2024-11-16T12:51:41,255 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1060): Cleaning up temporary data for 4d811d6d28d7d1423e02635376541525 2024-11-16T12:51:41,256 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=14 2024-11-16T12:51:41,257 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1093): writing seq id for 40aa1e14f3082cbed8ab6e2e88ab9432 2024-11-16T12:51:41,257 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1093): writing seq id for 4d811d6d28d7d1423e02635376541525 2024-11-16T12:51:41,258 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure b7ddacc6e0bab141a9de585b8ff7993a, server=0450ab8807f5,46757,1731761498722 in 232 msec 2024-11-16T12:51:41,259 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=13 2024-11-16T12:51:41,259 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure cb28d34770f7e0c8ce5cfa1e343a5743, server=0450ab8807f5,40215,1731761499880 in 223 msec 2024-11-16T12:51:41,260 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/40aa1e14f3082cbed8ab6e2e88ab9432/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:51:41,260 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=b7ddacc6e0bab141a9de585b8ff7993a, ASSIGN in 397 msec 2024-11-16T12:51:41,261 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/4d811d6d28d7d1423e02635376541525/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:51:41,261 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=cb28d34770f7e0c8ce5cfa1e343a5743, ASSIGN in 399 msec 2024-11-16T12:51:41,261 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1114): Opened 40aa1e14f3082cbed8ab6e2e88ab9432; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63475539, jitterRate=-0.054140761494636536}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T12:51:41,262 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 40aa1e14f3082cbed8ab6e2e88ab9432 2024-11-16T12:51:41,262 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1006): Region open journal for 40aa1e14f3082cbed8ab6e2e88ab9432: Running coprocessor pre-open hook at 1731761501248Writing region info on filesystem at 1731761501248Initializing all the Stores at 1731761501249 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761501249Cleaning up temporary data from old regions at 1731761501255 (+6 ms)Running coprocessor post-open hooks at 1731761501262 (+7 ms)Region opened successfully at 1731761501262 2024-11-16T12:51:41,262 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1114): Opened 4d811d6d28d7d1423e02635376541525; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74450351, jitterRate=0.10939668118953705}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T12:51:41,262 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4d811d6d28d7d1423e02635376541525 2024-11-16T12:51:41,262 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1006): Region open journal for 4d811d6d28d7d1423e02635376541525: Running coprocessor pre-open hook at 1731761501240Writing region info on filesystem at 1731761501240Initializing all the Stores at 1731761501241 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761501241Cleaning up temporary data from old regions at 1731761501255 (+14 ms)Running coprocessor post-open hooks at 1731761501262 (+7 ms)Region opened successfully at 1731761501262 2024-11-16T12:51:41,263 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2236): Post open deploy tasks for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1731761499980.4d811d6d28d7d1423e02635376541525., pid=20, masterSystemTime=1731761501183 2024-11-16T12:51:41,263 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegionServer(2236): Post open deploy tasks for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1731761499980.40aa1e14f3082cbed8ab6e2e88ab9432., pid=22, masterSystemTime=1731761501182 2024-11-16T12:51:41,265 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2266): Finished post open deploy task for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1731761499980.4d811d6d28d7d1423e02635376541525. 2024-11-16T12:51:41,265 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(153): Opened testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1731761499980.4d811d6d28d7d1423e02635376541525. 2024-11-16T12:51:41,265 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(132): Open testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1731761499980.d72159177cd75fa309be8f198cc5236c. 2024-11-16T12:51:41,265 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7752): Opening region: {ENCODED => d72159177cd75fa309be8f198cc5236c, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1731761499980.d72159177cd75fa309be8f198cc5236c.', STARTKEY => 'j', ENDKEY => ''} 2024-11-16T12:51:41,265 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRegionAssignmentAfterMasterRecoveryDueToZKExpiry d72159177cd75fa309be8f198cc5236c 2024-11-16T12:51:41,265 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1731761499980.d72159177cd75fa309be8f198cc5236c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:41,266 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7794): checking encryption for d72159177cd75fa309be8f198cc5236c 2024-11-16T12:51:41,266 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7797): checking classloading for d72159177cd75fa309be8f198cc5236c 2024-11-16T12:51:41,266 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=4d811d6d28d7d1423e02635376541525, regionState=OPEN, openSeqNum=2, regionLocation=0450ab8807f5,45113,1731761498684 2024-11-16T12:51:41,266 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegionServer(2266): Finished post open deploy task for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1731761499980.40aa1e14f3082cbed8ab6e2e88ab9432. 2024-11-16T12:51:41,266 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] handler.AssignRegionHandler(153): Opened testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1731761499980.40aa1e14f3082cbed8ab6e2e88ab9432. 2024-11-16T12:51:41,266 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1731761499980.3975dbb20d3790857ef1af53ef826e83. 2024-11-16T12:51:41,266 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 3975dbb20d3790857ef1af53ef826e83, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1731761499980.3975dbb20d3790857ef1af53ef826e83.', STARTKEY => 'b', ENDKEY => 'c'} 2024-11-16T12:51:41,267 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRegionAssignmentAfterMasterRecoveryDueToZKExpiry 3975dbb20d3790857ef1af53ef826e83 2024-11-16T12:51:41,267 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1731761499980.3975dbb20d3790857ef1af53ef826e83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:51:41,267 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 3975dbb20d3790857ef1af53ef826e83 2024-11-16T12:51:41,267 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 3975dbb20d3790857ef1af53ef826e83 2024-11-16T12:51:41,267 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=40aa1e14f3082cbed8ab6e2e88ab9432, regionState=OPEN, openSeqNum=2, regionLocation=0450ab8807f5,40215,1731761499880 2024-11-16T12:51:41,269 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d811d6d28d7d1423e02635376541525, server=0450ab8807f5,45113,1731761498684 because future has completed 2024-11-16T12:51:41,270 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40aa1e14f3082cbed8ab6e2e88ab9432, server=0450ab8807f5,40215,1731761499880 because future has completed 2024-11-16T12:51:41,271 INFO [StoreOpener-3975dbb20d3790857ef1af53ef826e83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 3975dbb20d3790857ef1af53ef826e83 2024-11-16T12:51:41,271 INFO [StoreOpener-d72159177cd75fa309be8f198cc5236c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region d72159177cd75fa309be8f198cc5236c 2024-11-16T12:51:41,273 INFO [StoreOpener-d72159177cd75fa309be8f198cc5236c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d72159177cd75fa309be8f198cc5236c columnFamilyName info 2024-11-16T12:51:41,273 INFO [StoreOpener-3975dbb20d3790857ef1af53ef826e83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3975dbb20d3790857ef1af53ef826e83 columnFamilyName info 2024-11-16T12:51:41,273 DEBUG [StoreOpener-3975dbb20d3790857ef1af53ef826e83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:41,273 DEBUG [StoreOpener-d72159177cd75fa309be8f198cc5236c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:51:41,273 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=12 2024-11-16T12:51:41,274 INFO [StoreOpener-d72159177cd75fa309be8f198cc5236c-1 {}] regionserver.HStore(327): Store=d72159177cd75fa309be8f198cc5236c/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:41,274 INFO [StoreOpener-3975dbb20d3790857ef1af53ef826e83-1 {}] regionserver.HStore(327): Store=3975dbb20d3790857ef1af53ef826e83/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:51:41,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 4d811d6d28d7d1423e02635376541525, server=0450ab8807f5,45113,1731761498684 in 248 msec 2024-11-16T12:51:41,274 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 3975dbb20d3790857ef1af53ef826e83 2024-11-16T12:51:41,275 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/3975dbb20d3790857ef1af53ef826e83 2024-11-16T12:51:41,275 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1038): replaying wal for d72159177cd75fa309be8f198cc5236c 2024-11-16T12:51:41,275 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/3975dbb20d3790857ef1af53ef826e83 2024-11-16T12:51:41,276 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 3975dbb20d3790857ef1af53ef826e83 2024-11-16T12:51:41,276 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 3975dbb20d3790857ef1af53ef826e83 2024-11-16T12:51:41,276 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/d72159177cd75fa309be8f198cc5236c 2024-11-16T12:51:41,276 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/d72159177cd75fa309be8f198cc5236c 2024-11-16T12:51:41,276 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=4d811d6d28d7d1423e02635376541525, ASSIGN in 414 msec 2024-11-16T12:51:41,277 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=11 2024-11-16T12:51:41,277 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 40aa1e14f3082cbed8ab6e2e88ab9432, server=0450ab8807f5,40215,1731761499880 in 245 msec 2024-11-16T12:51:41,277 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1048): stopping wal replay for d72159177cd75fa309be8f198cc5236c 2024-11-16T12:51:41,277 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1060): Cleaning up temporary data for d72159177cd75fa309be8f198cc5236c 2024-11-16T12:51:41,278 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 3975dbb20d3790857ef1af53ef826e83 2024-11-16T12:51:41,279 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=40aa1e14f3082cbed8ab6e2e88ab9432, ASSIGN in 417 msec 2024-11-16T12:51:41,279 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1093): writing seq id for d72159177cd75fa309be8f198cc5236c 2024-11-16T12:51:41,280 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/3975dbb20d3790857ef1af53ef826e83/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:51:41,281 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 3975dbb20d3790857ef1af53ef826e83; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69444652, jitterRate=0.03480595350265503}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T12:51:41,281 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/d72159177cd75fa309be8f198cc5236c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:51:41,281 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3975dbb20d3790857ef1af53ef826e83 2024-11-16T12:51:41,281 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 3975dbb20d3790857ef1af53ef826e83: Running coprocessor pre-open hook at 1731761501267Writing region info on filesystem at 1731761501267Initializing all the Stores at 1731761501268 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761501268Cleaning up temporary data from old regions at 1731761501276 (+8 ms)Running coprocessor post-open hooks at 1731761501281 (+5 ms)Region opened successfully at 1731761501281 2024-11-16T12:51:41,281 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1114): Opened d72159177cd75fa309be8f198cc5236c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70420208, jitterRate=0.04934287071228027}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T12:51:41,282 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d72159177cd75fa309be8f198cc5236c 2024-11-16T12:51:41,282 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1006): Region open journal for d72159177cd75fa309be8f198cc5236c: Running coprocessor pre-open hook at 1731761501266Writing region info on filesystem at 1731761501266Initializing all the Stores at 1731761501268 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761501268Cleaning up temporary data from old regions at 1731761501277 (+9 ms)Running coprocessor post-open hooks at 1731761501282 (+5 ms)Region opened successfully at 1731761501282 2024-11-16T12:51:41,282 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2236): Post open deploy tasks for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1731761499980.d72159177cd75fa309be8f198cc5236c., pid=19, masterSystemTime=1731761501183 2024-11-16T12:51:41,282 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1731761499980.3975dbb20d3790857ef1af53ef826e83., pid=24, masterSystemTime=1731761501182 2024-11-16T12:51:41,284 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1731761499980.3975dbb20d3790857ef1af53ef826e83. 2024-11-16T12:51:41,284 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1731761499980.3975dbb20d3790857ef1af53ef826e83. 2024-11-16T12:51:41,285 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3975dbb20d3790857ef1af53ef826e83, regionState=OPEN, openSeqNum=2, regionLocation=0450ab8807f5,40215,1731761499880 2024-11-16T12:51:41,285 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2266): Finished post open deploy task for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1731761499980.d72159177cd75fa309be8f198cc5236c. 2024-11-16T12:51:41,285 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(153): Opened testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1731761499980.d72159177cd75fa309be8f198cc5236c. 2024-11-16T12:51:41,287 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=d72159177cd75fa309be8f198cc5236c, regionState=OPEN, openSeqNum=2, regionLocation=0450ab8807f5,45113,1731761498684 2024-11-16T12:51:41,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3975dbb20d3790857ef1af53ef826e83, server=0450ab8807f5,40215,1731761499880 because future has completed 2024-11-16T12:51:41,289 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure d72159177cd75fa309be8f198cc5236c, server=0450ab8807f5,45113,1731761498684 because future has completed 2024-11-16T12:51:41,291 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=7 2024-11-16T12:51:41,291 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 3975dbb20d3790857ef1af53ef826e83, server=0450ab8807f5,40215,1731761499880 in 258 msec 2024-11-16T12:51:41,292 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=3975dbb20d3790857ef1af53ef826e83, ASSIGN in 431 msec 2024-11-16T12:51:41,292 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=15 2024-11-16T12:51:41,292 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure d72159177cd75fa309be8f198cc5236c, server=0450ab8807f5,45113,1731761498684 in 269 msec 2024-11-16T12:51:41,294 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=4 2024-11-16T12:51:41,294 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=d72159177cd75fa309be8f198cc5236c, ASSIGN in 432 msec 2024-11-16T12:51:41,295 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T12:51:41,295 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRegionAssignmentAfterMasterRecoveryDueToZKExpiry","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731761501295"}]},"ts":"1731761501295"} 2024-11-16T12:51:41,297 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, state=ENABLED in hbase:meta 2024-11-16T12:51:41,299 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T12:51:41,301 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry in 1.3190 sec 2024-11-16T12:51:42,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T12:51:42,130 INFO [RPCClient-NioEventLoopGroup-6-15 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRegionAssignmentAfterMasterRecoveryDueToZKExpiry completed 2024-11-16T12:51:42,130 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T12:51:42,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x10144fc6ca6001c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:42,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:42,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:42,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x10144fc6ca6001c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:42,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45113-0x10144fc6ca6001b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:42,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45113-0x10144fc6ca6001b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:42,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40215-0x10144fc6ca6001e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:42,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40215-0x10144fc6ca6001e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:42,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x10144fc6ca6001a, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:42,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x10144fc6ca6001a, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:42,186 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40215-0x10144fc6ca6001e, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:42,186 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46757-0x10144fc6ca6001c, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:42,187 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43165-0x10144fc6ca6001a, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:42,187 DEBUG [zk-event-processor-pool-0 {}] master.ActiveMasterManager(204): No master available. Notifying waiting threads 2024-11-16T12:51:42,187 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45113-0x10144fc6ca6001b, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:42,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x10144fc6ca6001a, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:42,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45113-0x10144fc6ca6001b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:42,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x10144fc6ca6001c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:42,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40215-0x10144fc6ca6001e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:42,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x10144fc6ca6001a, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:42,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45113-0x10144fc6ca6001b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:42,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x10144fc6ca6001c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:42,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40215-0x10144fc6ca6001e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:42,194 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43165-0x10144fc6ca6001a, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T12:51:42,194 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:43165-0x10144fc6ca6001a, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T12:51:42,195 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0450ab8807f5,43165,1731761498636 from backup master directory 2024-11-16T12:51:42,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x10144fc6ca6001c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:42,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40215-0x10144fc6ca6001e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:42,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45113-0x10144fc6ca6001b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:42,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x10144fc6ca6001a, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0450ab8807f5,43165,1731761498636 2024-11-16T12:51:42,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x10144fc6ca6001a, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:42,202 WARN [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T12:51:42,202 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0450ab8807f5,43165,1731761498636 2024-11-16T12:51:42,214 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:42,214 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T12:51:42,224 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=2) cost 10ms. 2024-11-16T12:51:42,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741934_1112 (size=196) 2024-11-16T12:51:42,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741934_1112 (size=196) 2024-11-16T12:51:42,275 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T12:51:42,275 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T12:51:42,275 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:51:42,280 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T12:51:42,281 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(342): old store file tracker DEFAULT is the same with new store file tracker, skip migration 2024-11-16T12:51:42,284 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(316): Renamed hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,39327,1731761498553 to hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,39327,1731761498553-dead as it is dead 2024-11-16T12:51:42,284 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,39327,1731761498553-dead/0450ab8807f5%2C39327%2C1731761498553.1731761498863 2024-11-16T12:51:42,285 WARN [IPC Server handler 0 on default port 44975 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,39327,1731761498553-dead/0450ab8807f5%2C39327%2C1731761498553.1731761498863 has not been closed. Lease recovery is in progress. RecoveryId = 1113 for block blk_1073741915_1093 2024-11-16T12:51:42,285 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,39327,1731761498553-dead/0450ab8807f5%2C39327%2C1731761498553.1731761498863 after 1ms 2024-11-16T12:51:42,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:42,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:42,287 ERROR [Time-limited test {}] master.HMaster(3218): Master server abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-16T12:51:42,287 ERROR [Time-limited test {}] master.HMaster(3223): ***** ABORTING master 0450ab8807f5,39327,1731761498553: Test recovery from zk session expired ***** org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired at org.apache.hadoop.hbase.TestZooKeeper.testRegionAssignmentAfterMasterRecoveryDueToZKExpiry(TestZooKeeper.java:196) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:42,288 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master '0450ab8807f5,39327,1731761498553' ***** 2024-11-16T12:51:42,288 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: Stopped by Time-limited test 2024-11-16T12:51:42,288 INFO [M:0;0450ab8807f5:39327 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T12:51:42,288 INFO [M:0;0450ab8807f5:39327 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;0450ab8807f5:39327. 2024-11-16T12:51:42,288 DEBUG [M:0;0450ab8807f5:39327 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:51:42,288 DEBUG [M:0;0450ab8807f5:39327 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:42,288 INFO [M:0;0450ab8807f5:39327 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T12:51:42,288 WARN [Time-limited test {}] zookeeper.ZKUtil(351): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/master org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /hbase/master at org.apache.zookeeper.KeeperException.create(KeeperException.java:134) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1972) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:368) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:356) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.getData(ZKUtil.java:343) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.MasterAddressTracker.getMasterAddress(MasterAddressTracker.java:171) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.ActiveMasterManager.stop(ActiveMasterManager.java:342) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.stop(HMaster.java:3327) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.stopMaster(HMaster.java:3315) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.abort(HMaster.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.TestZooKeeper.testRegionAssignmentAfterMasterRecoveryDueToZKExpiry(TestZooKeeper.java:196) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:42,288 DEBUG [M:0;0450ab8807f5:39327 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T12:51:42,288 DEBUG [M:0;0450ab8807f5:39327 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T12:51:42,288 ERROR [Time-limited test {}] zookeeper.ZKWatcher(711): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Received unexpected KeeperException, re-throwing exception org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /hbase/master at org.apache.zookeeper.KeeperException.create(KeeperException.java:134) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1972) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:368) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:356) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.getData(ZKUtil.java:343) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.MasterAddressTracker.getMasterAddress(MasterAddressTracker.java:171) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.ActiveMasterManager.stop(ActiveMasterManager.java:342) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.stop(HMaster.java:3327) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.stopMaster(HMaster.java:3315) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.abort(HMaster.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.TestZooKeeper.testRegionAssignmentAfterMasterRecoveryDueToZKExpiry(TestZooKeeper.java:196) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:42,288 INFO [M:0;0450ab8807f5:39327 {}] hbase.ChoreService(370): Chore service for: master/0450ab8807f5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T12:51:42,289 DEBUG [Time-limited test {}] master.ActiveMasterManager(353): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = Session expired for /hbase/master 2024-11-16T12:51:42,289 DEBUG [master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.large.0-1731761499086 {}] cleaner.HFileCleaner(306): Exit Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.large.0-1731761499086,5,FailOnTimeoutGroup] 2024-11-16T12:51:42,289 INFO [M:0;0450ab8807f5:39327 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T12:51:42,289 DEBUG [M:0;0450ab8807f5:39327 {}] master.HMaster(1795): Stopping service threads 2024-11-16T12:51:42,289 INFO [M:0;0450ab8807f5:39327 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T12:51:42,289 INFO [M:0;0450ab8807f5:39327 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T12:51:42,289 DEBUG [master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.small.0-1731761499086 {}] cleaner.HFileCleaner(306): Exit Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.small.0-1731761499086,5,FailOnTimeoutGroup] 2024-11-16T12:51:42,289 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T12:51:42,289 ERROR [M:0;0450ab8807f5:39327 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[ProcedureExecutor-Async-Task-Executor-3,10,PEWorkerGroup] Thread[ProcedureExecutor-Async-Task-Executor-5,10,PEWorkerGroup] Thread[ProcedureExecutor-Async-Task-Executor-7,10,PEWorkerGroup] 2024-11-16T12:51:42,289 ERROR [Time-limited test {}] master.HMaster(3225): ***** ABORTING master 0450ab8807f5,43165,1731761498636: killAll ***** 2024-11-16T12:51:42,289 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T12:51:42,290 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master '0450ab8807f5,43165,1731761498636' ***** 2024-11-16T12:51:42,290 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: Stopped by Time-limited test 2024-11-16T12:51:42,290 INFO [M:1;0450ab8807f5:43165 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T12:51:42,290 INFO [M:0;0450ab8807f5:39327 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=true 2024-11-16T12:51:42,290 INFO [M:1;0450ab8807f5:43165 {}] hbase.ChoreService(370): Chore service for: master/0450ab8807f5:0 had [] on shutdown 2024-11-16T12:51:42,290 INFO [M:1;0450ab8807f5:43165 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T12:51:42,290 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T12:51:42,290 DEBUG [M:1;0450ab8807f5:43165 {}] master.HMaster(1795): Stopping service threads 2024-11-16T12:51:42,290 WARN [M:0;0450ab8807f5:39327 {}] zookeeper.ZKUtil(351): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Unable to get data of znode /hbase/master org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /hbase/master at org.apache.zookeeper.KeeperException.create(KeeperException.java:134) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1972) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:368) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:356) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.getData(ZKUtil.java:343) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.MasterAddressTracker.getMasterAddress(MasterAddressTracker.java:171) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.ActiveMasterManager.stop(ActiveMasterManager.java:342) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.stopServiceThreads(HMaster.java:1809) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:631) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:42,290 ERROR [M:0;0450ab8807f5:39327 {}] zookeeper.ZKWatcher(711): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Received unexpected KeeperException, re-throwing exception org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /hbase/master at org.apache.zookeeper.KeeperException.create(KeeperException.java:134) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1972) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:368) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:356) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.getData(ZKUtil.java:343) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.MasterAddressTracker.getMasterAddress(MasterAddressTracker.java:171) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.ActiveMasterManager.stop(ActiveMasterManager.java:342) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.stopServiceThreads(HMaster.java:1809) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:631) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:42,290 DEBUG [M:0;0450ab8807f5:39327 {}] master.ActiveMasterManager(353): master:39327-0x10144fc6ca60019, quorum=127.0.0.1:53208, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = Session expired for /hbase/master 2024-11-16T12:51:42,291 INFO [M:0;0450ab8807f5:39327 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/.lastflushedseqids 2024-11-16T12:51:42,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741935_1114 (size=808) 2024-11-16T12:51:42,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741935_1114 (size=808) 2024-11-16T12:51:42,300 INFO [M:0;0450ab8807f5:39327 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T12:51:42,300 INFO [M:0;0450ab8807f5:39327 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=true 2024-11-16T12:51:42,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x10144fc6ca6001a, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:42,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45113-0x10144fc6ca6001b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:42,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x10144fc6ca6001a, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:42,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45113-0x10144fc6ca6001b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:42,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x10144fc6ca6001c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:42,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x10144fc6ca6001c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:42,302 INFO [M:1;0450ab8807f5:43165 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43165 2024-11-16T12:51:42,302 INFO [M:1;0450ab8807f5:43165 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T12:51:42,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40215-0x10144fc6ca6001e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:51:42,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40215-0x10144fc6ca6001e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:42,311 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45113-0x10144fc6ca6001b, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:42,311 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40215-0x10144fc6ca6001e, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:42,311 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46757-0x10144fc6ca6001c, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:51:42,311 DEBUG [Time-limited test {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-11-16T12:51:42,311 DEBUG [Time-limited test {}] master.ActiveMasterManager(353): master:43165-0x10144fc6ca6001a, quorum=127.0.0.1:53208, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-11-16T12:51:42,311 DEBUG [Time-limited test {}] master.HMaster(3213): Abort called but aborted=true, stopped=true 2024-11-16T12:51:42,312 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 0450ab8807f5,45113,1731761498684: killAll ***** 2024-11-16T12:51:42,312 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-16T12:51:42,312 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-16T12:51:42,312 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-16T12:51:42,312 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-16T12:51:42,312 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-16T12:51:42,313 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 284947872 }, "NonHeapMemoryUsage": { "committed": 171835392, "init": 7667712, "max": -1, "used": 169352512 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "0450ab8807f5", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [] } 2024-11-16T12:51:42,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741915_1093 (size=118281) 2024-11-16T12:51:42,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741915_1093 (size=118281) 2024-11-16T12:51:42,313 DEBUG [HMaster-EventLoopGroup-20-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0450ab8807f5,39327,1731761498553 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:42,313 WARN [Close-WAL-Writer-0 {}] wal.AsyncProtobufLogWriter(165): normal close failed, try recover java.io.IOException: Commit or complete block blk_1073741915_1093, whereas it is under recovery. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.commitOrCompleteLastBlock(BlockManager.java:1211) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.commitOrCompleteLastBlock(FSNamesystem.java:3890) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:731) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:163) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: Commit or complete block blk_1073741915_1093, whereas it is under recovery. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.commitOrCompleteLastBlock(BlockManager.java:1211) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.commitOrCompleteLastBlock(FSNamesystem.java:3890) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:731) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 6 more 2024-11-16T12:51:42,314 WARN [Time-limited test {}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.regionserver.RegionServerAbortedException): Server 0450ab8807f5,39327,1731761498553 aborting at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:?] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:?] at org.apache.hadoop.hbase.Abortable.abort(Abortable.java:45) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster.killAll(SingleProcessHBaseCluster.java:880) ~[test-classes/:?] at org.apache.hadoop.hbase.TestZooKeeper.after(TestZooKeeper.java:107) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: Server 0450ab8807f5,39327,1731761498553 aborting at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-16T12:51:42,315 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,39327,1731761498553/0450ab8807f5%2C39327%2C1731761498553.1731761498863 2024-11-16T12:51:42,316 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,39327,1731761498553/0450ab8807f5%2C39327%2C1731761498553.1731761498863 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.recoverLease(FSNamesystem.java:2869) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.recoverLease(NameNodeRpcServer.java:872) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.recoverLease(ClientNamenodeProtocolServerSideTranslatorPB.java:834) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:951) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.recoverAndClose(FanOutOneBlockAsyncDFSOutput.java:605) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:166) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,39327,1731761498553/0450ab8807f5%2C39327%2C1731761498553.1731761498863 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.recoverLease(FSNamesystem.java:2869) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.recoverLease(NameNodeRpcServer.java:872) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.recoverLease(ClientNamenodeProtocolServerSideTranslatorPB.java:834) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.recoverLease(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$recoverLease$24(ClientNamenodeProtocolTranslatorPB.java:685) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.recoverLease(ClientNamenodeProtocolTranslatorPB.java:685) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:949) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 17 more 2024-11-16T12:51:42,317 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44975/user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,39327,1731761498553/0450ab8807f5%2C39327%2C1731761498553.1731761498863 2024-11-16T12:51:42,318 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0450ab8807f5,45113,1731761498684' ***** 2024-11-16T12:51:42,319 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: killAll 2024-11-16T12:51:42,319 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 0450ab8807f5,46757,1731761498722: killAll ***** 2024-11-16T12:51:42,319 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-16T12:51:42,318 ERROR [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2033): Unable to recover lease after several attempts. Give up. java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,39327,1731761498553/0450ab8807f5%2C39327%2C1731761498553.1731761498863 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.recoverLease(FSNamesystem.java:2869) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.recoverLease(NameNodeRpcServer.java:872) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.recoverLease(ClientNamenodeProtocolServerSideTranslatorPB.java:834) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:951) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/MasterData/WALs/0450ab8807f5,39327,1731761498553/0450ab8807f5%2C39327%2C1731761498553.1731761498863 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.recoverLease(FSNamesystem.java:2869) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.recoverLease(NameNodeRpcServer.java:872) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.recoverLease(ClientNamenodeProtocolServerSideTranslatorPB.java:834) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.recoverLease(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$recoverLease$24(ClientNamenodeProtocolTranslatorPB.java:685) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.recoverLease(ClientNamenodeProtocolTranslatorPB.java:685) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor84.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:949) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 16 more 2024-11-16T12:51:42,319 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-16T12:51:42,319 INFO [RS:0;0450ab8807f5:45113 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T12:51:42,320 INFO [RS:0;0450ab8807f5:45113 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-16T12:51:42,320 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-16T12:51:42,320 INFO [RS:0;0450ab8807f5:45113 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-16T12:51:42,320 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-16T12:51:42,320 INFO [RS:0;0450ab8807f5:45113 {}] regionserver.HRegionServer(3091): Received CLOSE for 4d811d6d28d7d1423e02635376541525 2024-11-16T12:51:42,320 DEBUG [M:0;0450ab8807f5:39327 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T12:51:42,320 INFO [M:0;0450ab8807f5:39327 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:42,320 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-16T12:51:42,320 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T12:51:42,320 DEBUG [M:0;0450ab8807f5:39327 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:42,320 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 289142176 }, "NonHeapMemoryUsage": { "committed": 171835392, "init": 7667712, "max": -1, "used": 169361568 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "0450ab8807f5", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [] } 2024-11-16T12:51:42,320 DEBUG [M:0;0450ab8807f5:39327 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T12:51:42,320 DEBUG [M:0;0450ab8807f5:39327 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:42,320 INFO [RS:0;0450ab8807f5:45113 {}] regionserver.HRegionServer(3091): Received CLOSE for d72159177cd75fa309be8f198cc5236c 2024-11-16T12:51:42,320 INFO [RS:0;0450ab8807f5:45113 {}] regionserver.HRegionServer(3091): Received CLOSE for 4c4a6e6d05c0ed8942f754578d258c83 2024-11-16T12:51:42,320 INFO [RS:0;0450ab8807f5:45113 {}] regionserver.HRegionServer(3091): Received CLOSE for 1e95ff5fb519b8c6e236266576f123a3 2024-11-16T12:51:42,320 INFO [RS:0;0450ab8807f5:45113 {}] regionserver.HRegionServer(956): aborting server 0450ab8807f5,45113,1731761498684 2024-11-16T12:51:42,320 INFO [RS:0;0450ab8807f5:45113 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T12:51:42,320 INFO [RS:0;0450ab8807f5:45113 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0450ab8807f5:45113. 2024-11-16T12:51:42,320 DEBUG [RS:0;0450ab8807f5:45113 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:51:42,320 DEBUG [RS:0;0450ab8807f5:45113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:42,321 ERROR [M:0;0450ab8807f5:39327 {}] regionserver.HRegion(1960): Memstore data size is 104837 in region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:42,321 INFO [RS:0;0450ab8807f5:45113 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T12:51:42,321 INFO [M:0;0450ab8807f5:39327 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:51:42,321 INFO [RS:0;0450ab8807f5:45113 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T12:51:42,321 INFO [RS:0;0450ab8807f5:45113 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T12:51:42,321 DEBUG [M:0;0450ab8807f5:39327 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731761502320Disabling compacts and flushes for region at 1731761502320Disabling writes for close at 1731761502320Writing region close event to WAL at 1731761502320Closed at 1731761502321 (+1 ms) 2024-11-16T12:51:42,321 INFO [RS:0;0450ab8807f5:45113 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T12:51:42,321 INFO [M:0;0450ab8807f5:39327 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-16T12:51:42,321 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T12:51:42,321 INFO [M:0;0450ab8807f5:39327 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39327 2024-11-16T12:51:42,321 DEBUG [HMaster-EventLoopGroup-20-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0450ab8807f5,39327,1731761498553 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:42,322 INFO [M:0;0450ab8807f5:39327 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T12:51:42,322 INFO [M:0;0450ab8807f5:39327 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T12:51:42,322 WARN [Time-limited test {}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.regionserver.RegionServerAbortedException): Server 0450ab8807f5,39327,1731761498553 aborting at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:?] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:?] at org.apache.hadoop.hbase.Abortable.abort(Abortable.java:45) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster.killAll(SingleProcessHBaseCluster.java:880) ~[test-classes/:?] at org.apache.hadoop.hbase.TestZooKeeper.after(TestZooKeeper.java:107) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: Server 0450ab8807f5,39327,1731761498553 aborting at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-16T12:51:42,322 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4d811d6d28d7d1423e02635376541525, disabling compactions & flushes 2024-11-16T12:51:42,322 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1731761499980.4d811d6d28d7d1423e02635376541525. 2024-11-16T12:51:42,322 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1731761499980.4d811d6d28d7d1423e02635376541525. 2024-11-16T12:51:42,322 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1731761499980.4d811d6d28d7d1423e02635376541525. after waiting 0 ms 2024-11-16T12:51:42,322 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1731761499980.4d811d6d28d7d1423e02635376541525. 2024-11-16T12:51:42,322 INFO [RS:0;0450ab8807f5:45113 {}] regionserver.HRegionServer(1321): Waiting on 5 regions to close 2024-11-16T12:51:42,322 DEBUG [RS:0;0450ab8807f5:45113 {}] regionserver.HRegionServer(1325): Online Regions={4d811d6d28d7d1423e02635376541525=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1731761499980.4d811d6d28d7d1423e02635376541525., d72159177cd75fa309be8f198cc5236c=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1731761499980.d72159177cd75fa309be8f198cc5236c., 1588230740=hbase:meta,,1.1588230740, 4c4a6e6d05c0ed8942f754578d258c83=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1731761499980.4c4a6e6d05c0ed8942f754578d258c83., 1e95ff5fb519b8c6e236266576f123a3=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1731761499980.1e95ff5fb519b8c6e236266576f123a3.} 2024-11-16T12:51:42,322 DEBUG [RS:0;0450ab8807f5:45113 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1e95ff5fb519b8c6e236266576f123a3, 4c4a6e6d05c0ed8942f754578d258c83, 4d811d6d28d7d1423e02635376541525, d72159177cd75fa309be8f198cc5236c 2024-11-16T12:51:42,322 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T12:51:42,323 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T12:51:42,323 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T12:51:42,323 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T12:51:42,323 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T12:51:42,323 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1731761499980.4d811d6d28d7d1423e02635376541525. 2024-11-16T12:51:42,323 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4d811d6d28d7d1423e02635376541525: Waiting for close lock at 1731761502322Running coprocessor pre-close hooks at 1731761502322Disabling compacts and flushes for region at 1731761502322Disabling writes for close at 1731761502322Writing region close event to WAL at 1731761502323 (+1 ms)Running coprocessor post-close hooks at 1731761502323Closed at 1731761502323 2024-11-16T12:51:42,323 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1731761499980.4d811d6d28d7d1423e02635376541525. 2024-11-16T12:51:42,323 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d72159177cd75fa309be8f198cc5236c, disabling compactions & flushes 2024-11-16T12:51:42,323 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1731761499980.d72159177cd75fa309be8f198cc5236c. 2024-11-16T12:51:42,323 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1731761499980.d72159177cd75fa309be8f198cc5236c. 2024-11-16T12:51:42,323 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1731761499980.d72159177cd75fa309be8f198cc5236c. after waiting 0 ms 2024-11-16T12:51:42,323 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1731761499980.d72159177cd75fa309be8f198cc5236c. 2024-11-16T12:51:42,324 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0450ab8807f5,46757,1731761498722' ***** 2024-11-16T12:51:42,324 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: killAll 2024-11-16T12:51:42,324 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 0450ab8807f5,40215,1731761499880: killAll ***** 2024-11-16T12:51:42,324 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-16T12:51:42,324 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-16T12:51:42,324 INFO [RS:1;0450ab8807f5:46757 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T12:51:42,324 INFO [RS:1;0450ab8807f5:46757 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-16T12:51:42,324 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T12:51:42,324 INFO [RS:1;0450ab8807f5:46757 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-16T12:51:42,324 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-16T12:51:42,324 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-16T12:51:42,324 INFO [RS:1;0450ab8807f5:46757 {}] regionserver.HRegionServer(3091): Received CLOSE for ca39b850f78d2f0a8e65d12ae3aff16c 2024-11-16T12:51:42,324 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-16T12:51:42,324 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 291239328 }, "NonHeapMemoryUsage": { "committed": 171900928, "init": 7667712, "max": -1, "used": 169391568 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "0450ab8807f5", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [] } 2024-11-16T12:51:42,325 ERROR [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 18827 in region hbase:meta,,1.1588230740 2024-11-16T12:51:42,325 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T12:51:42,325 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T12:51:42,325 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731761502322Running coprocessor pre-close hooks at 1731761502322Disabling compacts and flushes for region at 1731761502322Disabling writes for close at 1731761502323 (+1 ms)Writing region close event to WAL at 1731761502325 (+2 ms)Running coprocessor post-close hooks at 1731761502325Closed at 1731761502325 2024-11-16T12:51:42,325 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T12:51:42,325 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-24-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 0450ab8807f5:39327 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0450ab8807f5/172.17.0.2:39327 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:51:42,326 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-24-2 {}] ipc.FailedServers(52): Added failed server with address 0450ab8807f5:39327 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0450ab8807f5/172.17.0.2:39327 2024-11-16T12:51:42,326 WARN [Time-limited test {}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: java.net.ConnectException: Call to address=0450ab8807f5:39327 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0450ab8807f5/172.17.0.2:39327 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:?] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:?] at org.apache.hadoop.hbase.Abortable.abort(Abortable.java:45) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster.killAll(SingleProcessHBaseCluster.java:880) ~[test-classes/:?] at org.apache.hadoop.hbase.TestZooKeeper.after(TestZooKeeper.java:107) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=0450ab8807f5:39327 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0450ab8807f5/172.17.0.2:39327 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0450ab8807f5/172.17.0.2:39327 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-16T12:51:42,330 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1731761499980.d72159177cd75fa309be8f198cc5236c. 2024-11-16T12:51:42,330 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d72159177cd75fa309be8f198cc5236c: Waiting for close lock at 1731761502323Running coprocessor pre-close hooks at 1731761502323Disabling compacts and flushes for region at 1731761502323Disabling writes for close at 1731761502323Writing region close event to WAL at 1731761502330 (+7 ms)Running coprocessor post-close hooks at 1731761502330Closed at 1731761502330 2024-11-16T12:51:42,330 INFO [RS:1;0450ab8807f5:46757 {}] regionserver.HRegionServer(3091): Received CLOSE for b7ddacc6e0bab141a9de585b8ff7993a 2024-11-16T12:51:42,330 INFO [RS:1;0450ab8807f5:46757 {}] regionserver.HRegionServer(3091): Received CLOSE for 454d6867041c8b378ebdd3f87fca5f11 2024-11-16T12:51:42,330 INFO [RS:1;0450ab8807f5:46757 {}] regionserver.HRegionServer(956): aborting server 0450ab8807f5,46757,1731761498722 2024-11-16T12:51:42,330 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1731761499980.d72159177cd75fa309be8f198cc5236c. 2024-11-16T12:51:42,330 INFO [RS:1;0450ab8807f5:46757 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T12:51:42,331 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0450ab8807f5,40215,1731761499880' ***** 2024-11-16T12:51:42,331 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ca39b850f78d2f0a8e65d12ae3aff16c, disabling compactions & flushes 2024-11-16T12:51:42,331 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: killAll 2024-11-16T12:51:42,331 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1731761499980.ca39b850f78d2f0a8e65d12ae3aff16c. 2024-11-16T12:51:42,331 INFO [RS:1;0450ab8807f5:46757 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;0450ab8807f5:46757. 2024-11-16T12:51:42,331 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4c4a6e6d05c0ed8942f754578d258c83, disabling compactions & flushes 2024-11-16T12:51:42,331 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1731761499980.ca39b850f78d2f0a8e65d12ae3aff16c. 2024-11-16T12:51:42,331 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1731761499980.4c4a6e6d05c0ed8942f754578d258c83. 2024-11-16T12:51:42,331 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1731761499980.ca39b850f78d2f0a8e65d12ae3aff16c. after waiting 0 ms 2024-11-16T12:51:42,331 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1731761499980.4c4a6e6d05c0ed8942f754578d258c83. 2024-11-16T12:51:42,331 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1731761499980.4c4a6e6d05c0ed8942f754578d258c83. after waiting 0 ms 2024-11-16T12:51:42,331 INFO [RS:2;0450ab8807f5:40215 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T12:51:42,331 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1731761499980.4c4a6e6d05c0ed8942f754578d258c83. 2024-11-16T12:51:42,331 INFO [RS:2;0450ab8807f5:40215 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-16T12:51:42,331 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T12:51:42,331 INFO [RS:2;0450ab8807f5:40215 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-16T12:51:42,331 INFO [RS:2;0450ab8807f5:40215 {}] regionserver.HRegionServer(3091): Received CLOSE for 40aa1e14f3082cbed8ab6e2e88ab9432 2024-11-16T12:51:42,331 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T12:51:42,331 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1731761499980.ca39b850f78d2f0a8e65d12ae3aff16c. 2024-11-16T12:51:42,331 DEBUG [RS:1;0450ab8807f5:46757 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:51:42,331 DEBUG [RS:1;0450ab8807f5:46757 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:42,331 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.TestZooKeeper.after(TestZooKeeper.java:110) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:51:42,331 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:42,331 INFO [RS:1;0450ab8807f5:46757 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-11-16T12:51:42,331 DEBUG [RS:1;0450ab8807f5:46757 {}] regionserver.HRegionServer(1325): Online Regions={ca39b850f78d2f0a8e65d12ae3aff16c=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1731761499980.ca39b850f78d2f0a8e65d12ae3aff16c., b7ddacc6e0bab141a9de585b8ff7993a=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1731761499980.b7ddacc6e0bab141a9de585b8ff7993a., 454d6867041c8b378ebdd3f87fca5f11=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1731761499980.454d6867041c8b378ebdd3f87fca5f11.} 2024-11-16T12:51:42,331 DEBUG [RS:1;0450ab8807f5:46757 {}] regionserver.HRegionServer(1351): Waiting on 454d6867041c8b378ebdd3f87fca5f11, b7ddacc6e0bab141a9de585b8ff7993a, ca39b850f78d2f0a8e65d12ae3aff16c 2024-11-16T12:51:42,332 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6081c7f0 to 127.0.0.1:53208 2024-11-16T12:51:42,332 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-16T12:51:42,333 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1731761499980.4c4a6e6d05c0ed8942f754578d258c83. 2024-11-16T12:51:42,333 INFO [RS:2;0450ab8807f5:40215 {}] regionserver.HRegionServer(3091): Received CLOSE for 3975dbb20d3790857ef1af53ef826e83 2024-11-16T12:51:42,333 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4c4a6e6d05c0ed8942f754578d258c83: Waiting for close lock at 1731761502331Running coprocessor pre-close hooks at 1731761502331Disabling compacts and flushes for region at 1731761502331Disabling writes for close at 1731761502331Writing region close event to WAL at 1731761502332 (+1 ms)Running coprocessor post-close hooks at 1731761502332Closed at 1731761502333 (+1 ms) 2024-11-16T12:51:42,333 INFO [RS:2;0450ab8807f5:40215 {}] regionserver.HRegionServer(3091): Received CLOSE for cb28d34770f7e0c8ce5cfa1e343a5743 2024-11-16T12:51:42,333 INFO [RS:2;0450ab8807f5:40215 {}] regionserver.HRegionServer(3091): Received CLOSE for fed5462f2af6a59954ccad57d1eb36f1 2024-11-16T12:51:42,333 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 40aa1e14f3082cbed8ab6e2e88ab9432, disabling compactions & flushes 2024-11-16T12:51:42,333 INFO [RS:2;0450ab8807f5:40215 {}] regionserver.HRegionServer(956): aborting server 0450ab8807f5,40215,1731761499880 2024-11-16T12:51:42,333 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1731761499980.4c4a6e6d05c0ed8942f754578d258c83. 2024-11-16T12:51:42,333 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1731761499980.40aa1e14f3082cbed8ab6e2e88ab9432. 2024-11-16T12:51:42,333 INFO [RS:2;0450ab8807f5:40215 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T12:51:42,333 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1731761499980.40aa1e14f3082cbed8ab6e2e88ab9432. 2024-11-16T12:51:42,333 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1731761499980.40aa1e14f3082cbed8ab6e2e88ab9432. after waiting 0 ms 2024-11-16T12:51:42,333 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1731761499980.40aa1e14f3082cbed8ab6e2e88ab9432. 2024-11-16T12:51:42,333 INFO [RS:2;0450ab8807f5:40215 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;0450ab8807f5:40215. 2024-11-16T12:51:42,333 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1e95ff5fb519b8c6e236266576f123a3, disabling compactions & flushes 2024-11-16T12:51:42,333 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1731761499980.1e95ff5fb519b8c6e236266576f123a3. 2024-11-16T12:51:42,333 DEBUG [RS:2;0450ab8807f5:40215 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:51:42,333 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1731761499980.1e95ff5fb519b8c6e236266576f123a3. 2024-11-16T12:51:42,333 DEBUG [RS:2;0450ab8807f5:40215 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:42,333 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1731761499980.1e95ff5fb519b8c6e236266576f123a3. after waiting 0 ms 2024-11-16T12:51:42,333 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1731761499980.1e95ff5fb519b8c6e236266576f123a3. 2024-11-16T12:51:42,333 INFO [RS:2;0450ab8807f5:40215 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-11-16T12:51:42,333 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1731761499980.ca39b850f78d2f0a8e65d12ae3aff16c. 2024-11-16T12:51:42,333 DEBUG [RS:2;0450ab8807f5:40215 {}] regionserver.HRegionServer(1325): Online Regions={40aa1e14f3082cbed8ab6e2e88ab9432=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1731761499980.40aa1e14f3082cbed8ab6e2e88ab9432., 3975dbb20d3790857ef1af53ef826e83=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1731761499980.3975dbb20d3790857ef1af53ef826e83., cb28d34770f7e0c8ce5cfa1e343a5743=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1731761499980.cb28d34770f7e0c8ce5cfa1e343a5743., fed5462f2af6a59954ccad57d1eb36f1=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1731761499980.fed5462f2af6a59954ccad57d1eb36f1.} 2024-11-16T12:51:42,333 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ca39b850f78d2f0a8e65d12ae3aff16c: Waiting for close lock at 1731761502330Running coprocessor pre-close hooks at 1731761502330Disabling compacts and flushes for region at 1731761502330Disabling writes for close at 1731761502331 (+1 ms)Writing region close event to WAL at 1731761502333 (+2 ms)Running coprocessor post-close hooks at 1731761502333Closed at 1731761502333 2024-11-16T12:51:42,333 DEBUG [RS:2;0450ab8807f5:40215 {}] regionserver.HRegionServer(1351): Waiting on 3975dbb20d3790857ef1af53ef826e83, 40aa1e14f3082cbed8ab6e2e88ab9432, cb28d34770f7e0c8ce5cfa1e343a5743, fed5462f2af6a59954ccad57d1eb36f1 2024-11-16T12:51:42,333 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1731761499980.ca39b850f78d2f0a8e65d12ae3aff16c. 2024-11-16T12:51:42,333 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b7ddacc6e0bab141a9de585b8ff7993a, disabling compactions & flushes 2024-11-16T12:51:42,333 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1731761499980.b7ddacc6e0bab141a9de585b8ff7993a. 2024-11-16T12:51:42,333 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1731761499980.b7ddacc6e0bab141a9de585b8ff7993a. 2024-11-16T12:51:42,333 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1731761499980.b7ddacc6e0bab141a9de585b8ff7993a. after waiting 0 ms 2024-11-16T12:51:42,333 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1731761499980.b7ddacc6e0bab141a9de585b8ff7993a. 2024-11-16T12:51:42,334 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1731761499980.40aa1e14f3082cbed8ab6e2e88ab9432. 2024-11-16T12:51:42,334 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 40aa1e14f3082cbed8ab6e2e88ab9432: Waiting for close lock at 1731761502333Running coprocessor pre-close hooks at 1731761502333Disabling compacts and flushes for region at 1731761502333Disabling writes for close at 1731761502333Writing region close event to WAL at 1731761502334 (+1 ms)Running coprocessor post-close hooks at 1731761502334Closed at 1731761502334 2024-11-16T12:51:42,335 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1731761499980.40aa1e14f3082cbed8ab6e2e88ab9432. 2024-11-16T12:51:42,335 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3975dbb20d3790857ef1af53ef826e83, disabling compactions & flushes 2024-11-16T12:51:42,335 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1731761499980.3975dbb20d3790857ef1af53ef826e83. 2024-11-16T12:51:42,335 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1731761499980.3975dbb20d3790857ef1af53ef826e83. 2024-11-16T12:51:42,335 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1731761499980.3975dbb20d3790857ef1af53ef826e83. after waiting 0 ms 2024-11-16T12:51:42,335 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1731761499980.3975dbb20d3790857ef1af53ef826e83. 2024-11-16T12:51:42,339 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1731761499980.b7ddacc6e0bab141a9de585b8ff7993a. 2024-11-16T12:51:42,339 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1731761499980.1e95ff5fb519b8c6e236266576f123a3. 2024-11-16T12:51:42,339 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1e95ff5fb519b8c6e236266576f123a3: Waiting for close lock at 1731761502333Running coprocessor pre-close hooks at 1731761502333Disabling compacts and flushes for region at 1731761502333Disabling writes for close at 1731761502333Writing region close event to WAL at 1731761502339 (+6 ms)Running coprocessor post-close hooks at 1731761502339Closed at 1731761502339 2024-11-16T12:51:42,339 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b7ddacc6e0bab141a9de585b8ff7993a: Waiting for close lock at 1731761502333Running coprocessor pre-close hooks at 1731761502333Disabling compacts and flushes for region at 1731761502333Disabling writes for close at 1731761502333Writing region close event to WAL at 1731761502339 (+6 ms)Running coprocessor post-close hooks at 1731761502339Closed at 1731761502339 2024-11-16T12:51:42,339 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1731761499980.3975dbb20d3790857ef1af53ef826e83. 2024-11-16T12:51:42,339 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1731761499980.b7ddacc6e0bab141a9de585b8ff7993a. 2024-11-16T12:51:42,339 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1731761499980.1e95ff5fb519b8c6e236266576f123a3. 2024-11-16T12:51:42,339 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3975dbb20d3790857ef1af53ef826e83: Waiting for close lock at 1731761502335Running coprocessor pre-close hooks at 1731761502335Disabling compacts and flushes for region at 1731761502335Disabling writes for close at 1731761502335Writing region close event to WAL at 1731761502339 (+4 ms)Running coprocessor post-close hooks at 1731761502339Closed at 1731761502339 2024-11-16T12:51:42,339 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1731761499980.3975dbb20d3790857ef1af53ef826e83. 2024-11-16T12:51:42,339 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 454d6867041c8b378ebdd3f87fca5f11, disabling compactions & flushes 2024-11-16T12:51:42,339 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1731761499980.454d6867041c8b378ebdd3f87fca5f11. 2024-11-16T12:51:42,339 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cb28d34770f7e0c8ce5cfa1e343a5743, disabling compactions & flushes 2024-11-16T12:51:42,339 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1731761499980.cb28d34770f7e0c8ce5cfa1e343a5743. 2024-11-16T12:51:42,339 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1731761499980.454d6867041c8b378ebdd3f87fca5f11. 2024-11-16T12:51:42,339 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1731761499980.cb28d34770f7e0c8ce5cfa1e343a5743. 2024-11-16T12:51:42,339 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1731761499980.454d6867041c8b378ebdd3f87fca5f11. after waiting 0 ms 2024-11-16T12:51:42,339 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1731761499980.cb28d34770f7e0c8ce5cfa1e343a5743. after waiting 0 ms 2024-11-16T12:51:42,339 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1731761499980.454d6867041c8b378ebdd3f87fca5f11. 2024-11-16T12:51:42,339 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1731761499980.cb28d34770f7e0c8ce5cfa1e343a5743. 2024-11-16T12:51:42,340 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1731761499980.454d6867041c8b378ebdd3f87fca5f11. 2024-11-16T12:51:42,340 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1731761499980.cb28d34770f7e0c8ce5cfa1e343a5743. 2024-11-16T12:51:42,340 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 454d6867041c8b378ebdd3f87fca5f11: Waiting for close lock at 1731761502339Running coprocessor pre-close hooks at 1731761502339Disabling compacts and flushes for region at 1731761502339Disabling writes for close at 1731761502339Writing region close event to WAL at 1731761502340 (+1 ms)Running coprocessor post-close hooks at 1731761502340Closed at 1731761502340 2024-11-16T12:51:42,340 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cb28d34770f7e0c8ce5cfa1e343a5743: Waiting for close lock at 1731761502339Running coprocessor pre-close hooks at 1731761502339Disabling compacts and flushes for region at 1731761502339Disabling writes for close at 1731761502339Writing region close event to WAL at 1731761502340 (+1 ms)Running coprocessor post-close hooks at 1731761502340Closed at 1731761502340 2024-11-16T12:51:42,340 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1731761499980.454d6867041c8b378ebdd3f87fca5f11. 2024-11-16T12:51:42,340 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1731761499980.cb28d34770f7e0c8ce5cfa1e343a5743. 2024-11-16T12:51:42,340 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fed5462f2af6a59954ccad57d1eb36f1, disabling compactions & flushes 2024-11-16T12:51:42,340 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1731761499980.fed5462f2af6a59954ccad57d1eb36f1. 2024-11-16T12:51:42,340 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1731761499980.fed5462f2af6a59954ccad57d1eb36f1. 2024-11-16T12:51:42,340 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1731761499980.fed5462f2af6a59954ccad57d1eb36f1. after waiting 0 ms 2024-11-16T12:51:42,340 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1731761499980.fed5462f2af6a59954ccad57d1eb36f1. 2024-11-16T12:51:42,340 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1731761499980.fed5462f2af6a59954ccad57d1eb36f1. 2024-11-16T12:51:42,340 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fed5462f2af6a59954ccad57d1eb36f1: Waiting for close lock at 1731761502340Running coprocessor pre-close hooks at 1731761502340Disabling compacts and flushes for region at 1731761502340Disabling writes for close at 1731761502340Writing region close event to WAL at 1731761502340Running coprocessor post-close hooks at 1731761502340Closed at 1731761502340 2024-11-16T12:51:42,340 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1731761499980.fed5462f2af6a59954ccad57d1eb36f1. 2024-11-16T12:51:42,367 INFO [regionserver/0450ab8807f5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:51:42,367 INFO [regionserver/0450ab8807f5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:51:42,368 INFO [regionserver/0450ab8807f5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:51:42,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x10144fc6ca6001a, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:42,419 INFO [M:1;0450ab8807f5:43165 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T12:51:42,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43165-0x10144fc6ca6001a, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:42,523 INFO [RS:0;0450ab8807f5:45113 {}] regionserver.HRegionServer(976): stopping server 0450ab8807f5,45113,1731761498684; all regions closed. 2024-11-16T12:51:42,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741920_1098 (size=20589) 2024-11-16T12:51:42,525 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,45113,1731761498684/0450ab8807f5%2C45113%2C1731761498684.meta.1731761499738.meta not finished, retry = 0 2024-11-16T12:51:42,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741920_1098 (size=20589) 2024-11-16T12:51:42,532 INFO [RS:1;0450ab8807f5:46757 {}] regionserver.HRegionServer(976): stopping server 0450ab8807f5,46757,1731761498722; all regions closed. 2024-11-16T12:51:42,533 INFO [RS:2;0450ab8807f5:40215 {}] regionserver.HRegionServer(976): stopping server 0450ab8807f5,40215,1731761499880; all regions closed. 2024-11-16T12:51:42,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741918_1096 (size=1305) 2024-11-16T12:51:42,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741918_1096 (size=1305) 2024-11-16T12:51:42,535 DEBUG [RS:1;0450ab8807f5:46757 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:42,535 INFO [RS:1;0450ab8807f5:46757 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:51:42,535 INFO [RS:1;0450ab8807f5:46757 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T12:51:42,535 INFO [RS:1;0450ab8807f5:46757 {}] hbase.ChoreService(370): Chore service for: regionserver/0450ab8807f5:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T12:51:42,536 INFO [RS:1;0450ab8807f5:46757 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T12:51:42,536 INFO [regionserver/0450ab8807f5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T12:51:42,536 INFO [RS:1;0450ab8807f5:46757 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T12:51:42,536 INFO [RS:1;0450ab8807f5:46757 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T12:51:42,536 INFO [RS:1;0450ab8807f5:46757 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T12:51:42,536 INFO [RS:1;0450ab8807f5:46757 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46757 2024-11-16T12:51:42,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741926_1104 (size=1709) 2024-11-16T12:51:42,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741926_1104 (size=1709) 2024-11-16T12:51:42,537 DEBUG [RS:2;0450ab8807f5:40215 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:42,537 INFO [RS:2;0450ab8807f5:40215 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:51:42,537 INFO [RS:2;0450ab8807f5:40215 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T12:51:42,537 INFO [RS:2;0450ab8807f5:40215 {}] hbase.ChoreService(370): Chore service for: regionserver/0450ab8807f5:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-16T12:51:42,537 INFO [RS:2;0450ab8807f5:40215 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T12:51:42,537 INFO [regionserver/0450ab8807f5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T12:51:42,538 INFO [RS:2;0450ab8807f5:40215 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T12:51:42,538 INFO [RS:2;0450ab8807f5:40215 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T12:51:42,538 INFO [RS:2;0450ab8807f5:40215 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T12:51:42,538 INFO [RS:2;0450ab8807f5:40215 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40215 2024-11-16T12:51:42,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x10144fc6ca6001c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0450ab8807f5,46757,1731761498722 2024-11-16T12:51:42,559 INFO [RS:1;0450ab8807f5:46757 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T12:51:42,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40215-0x10144fc6ca6001e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0450ab8807f5,40215,1731761499880 2024-11-16T12:51:42,569 INFO [RS:2;0450ab8807f5:40215 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T12:51:42,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43257 is added to blk_1073741919_1097 (size=1708) 2024-11-16T12:51:42,628 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/f8816cb4-0e78-4a64-fd75-be989163722b/WALs/0450ab8807f5,45113,1731761498684/0450ab8807f5%2C45113%2C1731761498684.1731761499382 not finished, retry = 0 2024-11-16T12:51:42,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36961 is added to blk_1073741919_1097 (size=1708) 2024-11-16T12:51:42,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x10144fc6ca6001c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:42,678 INFO [RS:1;0450ab8807f5:46757 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T12:51:42,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x10144fc6ca6001c, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:42,678 INFO [RS:1;0450ab8807f5:46757 {}] regionserver.HRegionServer(1031): Exiting; stopping=0450ab8807f5,46757,1731761498722; zookeeper connection closed. 2024-11-16T12:51:42,678 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@b6baad8 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@b6baad8 2024-11-16T12:51:42,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40215-0x10144fc6ca6001e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:42,686 INFO [RS:2;0450ab8807f5:40215 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T12:51:42,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40215-0x10144fc6ca6001e, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:42,686 INFO [RS:2;0450ab8807f5:40215 {}] regionserver.HRegionServer(1031): Exiting; stopping=0450ab8807f5,40215,1731761499880; zookeeper connection closed. 2024-11-16T12:51:42,686 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1ffe23d1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1ffe23d1 2024-11-16T12:51:42,729 DEBUG [RS:0;0450ab8807f5:45113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:51:42,729 INFO [RS:0;0450ab8807f5:45113 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:51:42,730 INFO [RS:0;0450ab8807f5:45113 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T12:51:42,730 INFO [RS:0;0450ab8807f5:45113 {}] hbase.ChoreService(370): Chore service for: regionserver/0450ab8807f5:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-16T12:51:42,730 INFO [RS:0;0450ab8807f5:45113 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T12:51:42,730 INFO [regionserver/0450ab8807f5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T12:51:42,730 INFO [RS:0;0450ab8807f5:45113 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45113 2024-11-16T12:51:42,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45113-0x10144fc6ca6001b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0450ab8807f5,45113,1731761498684 2024-11-16T12:51:42,744 INFO [RS:0;0450ab8807f5:45113 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T12:51:42,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45113-0x10144fc6ca6001b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:42,852 INFO [RS:0;0450ab8807f5:45113 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T12:51:42,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45113-0x10144fc6ca6001b, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:42,852 INFO [RS:0;0450ab8807f5:45113 {}] regionserver.HRegionServer(1031): Exiting; stopping=0450ab8807f5,45113,1731761498684; zookeeper connection closed. 2024-11-16T12:51:42,853 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@39062f06 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@39062f06 2024-11-16T12:51:42,853 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 2 master(s) and 3 regionserver(s) complete 2024-11-16T12:51:42,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca60018, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:42,978 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T12:51:42,981 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=testing utility connecting to ZooKeeper ensemble=127.0.0.1:53208 2024-11-16T12:51:43,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility0x0, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:51:43,001 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): testing utility-0x10144fc6ca6001f connected 2024-11-16T12:51:43,001 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-16T12:51:43,001 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/meta-region-server 2024-11-16T12:51:43,002 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/rs 2024-11-16T12:51:43,002 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/splitWAL 2024-11-16T12:51:43,003 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters 2024-11-16T12:51:43,003 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc 2024-11-16T12:51:43,003 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/draining 2024-11-16T12:51:43,003 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/master-maintenance 2024-11-16T12:51:43,004 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot 2024-11-16T12:51:43,004 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/hbaseid 2024-11-16T12:51:43,004 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/table 2024-11-16T12:51:43,007 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc/abort 2024-11-16T12:51:43,007 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc/acquired 2024-11-16T12:51:43,007 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc/reached 2024-11-16T12:51:43,008 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot/abort 2024-11-16T12:51:43,009 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot/acquired 2024-11-16T12:51:43,009 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot/reached 2024-11-16T12:51:43,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/reached 2024-11-16T12:51:43,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/online-snapshot 2024-11-16T12:51:43,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/acquired 2024-11-16T12:51:43,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/abort 2024-11-16T12:51:43,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/reached 2024-11-16T12:51:43,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/flush-table-proc 2024-11-16T12:51:43,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/acquired 2024-11-16T12:51:43,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/abort 2024-11-16T12:51:43,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/table 2024-11-16T12:51:43,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:51:43,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/hbaseid 2024-11-16T12:51:43,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot 2024-11-16T12:51:43,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master-maintenance 2024-11-16T12:51:43,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/draining 2024-11-16T12:51:43,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc 2024-11-16T12:51:43,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:51:43,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/splitWAL 2024-11-16T12:51:43,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs 2024-11-16T12:51:43,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:51:43,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T12:51:43,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase 2024-11-16T12:51:43,031 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestZooKeeper#testRegionAssignmentAfterMasterRecoveryDueToZKExpiry Thread=436 (was 403) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-24-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-21-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-23-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1205652929) connection to localhost/127.0.0.1:44975 from jenkins.hfs.9 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1205652929) connection to localhost/127.0.0.1:44975 from jenkins.hfs.11 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-23-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-20-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-22-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1205652929) connection to localhost/127.0.0.1:44975 from jenkins.hfs.10 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1694157692_22 at /127.0.0.1:40774 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-20-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.10@localhost:44975 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master:store-WAL-Roller java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:180) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-22-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/0450ab8807f5:0:becomeActiveMaster java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:166) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:82) app//org.apache.hadoop.hbase.master.region.MasterRegion.replayWALs(MasterRegion.java:323) app//org.apache.hadoop.hbase.master.region.MasterRegion.open(MasterRegion.java:280) app//org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:449) app//org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) app//org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) app//org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) app//org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) app//org.apache.hadoop.hbase.master.HMaster$$Lambda$383/0x00007f9bb0935400.run(Unknown Source) app//org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) app//org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$364/0x00007f9bb08dfc60.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.11@localhost:44975 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1694157692_22 at /127.0.0.1:48598 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-24-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.9@localhost:44975 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-23-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-24-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-22-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1694157692_22 at /127.0.0.1:50302 [Waiting for operation #57] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-20-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=742 (was 687) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=216 (was 226), ProcessCount=11 (was 11), AvailableMemoryMB=3089 (was 3165) 2024-11-16T12:51:43,031 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-16T12:51:43,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10144fc6ca6001f, quorum=127.0.0.1:53208, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:51:43,209 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1a3128fc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T12:51:43,211 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8ad07d8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T12:51:43,211 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T12:51:43,211 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@101ec660{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T12:51:43,211 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@259b06a7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/hadoop.log.dir/,STOPPED} 2024-11-16T12:51:43,213 WARN [BP-1933630982-172.17.0.2-1731761461817 heartbeating to localhost/127.0.0.1:44975 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-16T12:51:43,213 WARN [BP-1933630982-172.17.0.2-1731761461817 heartbeating to localhost/127.0.0.1:44975 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1933630982-172.17.0.2-1731761461817 (Datanode Uuid 6d7ed518-8338-4c85-a9a9-e6a32b4a2a95) service to localhost/127.0.0.1:44975 2024-11-16T12:51:43,213 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-16T12:51:43,214 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-16T12:51:43,215 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/cluster_35bff147-7ae7-a41a-0bf2-848ec5006277/data/data3/current/BP-1933630982-172.17.0.2-1731761461817 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T12:51:43,215 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/cluster_35bff147-7ae7-a41a-0bf2-848ec5006277/data/data4/current/BP-1933630982-172.17.0.2-1731761461817 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T12:51:43,215 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-16T12:51:43,217 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@602b1375{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T12:51:43,217 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@78ddb114{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T12:51:43,218 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T12:51:43,218 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@44042549{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T12:51:43,218 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@13ff53d8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/hadoop.log.dir/,STOPPED} 2024-11-16T12:51:43,219 WARN [BP-1933630982-172.17.0.2-1731761461817 heartbeating to localhost/127.0.0.1:44975 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-16T12:51:43,219 WARN [BP-1933630982-172.17.0.2-1731761461817 heartbeating to localhost/127.0.0.1:44975 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1933630982-172.17.0.2-1731761461817 (Datanode Uuid e55698f0-e627-47fd-9d4b-3c2a6949c890) service to localhost/127.0.0.1:44975 2024-11-16T12:51:43,219 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-16T12:51:43,219 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-16T12:51:43,219 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/cluster_35bff147-7ae7-a41a-0bf2-848ec5006277/data/data1/current/BP-1933630982-172.17.0.2-1731761461817 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T12:51:43,220 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/cluster_35bff147-7ae7-a41a-0bf2-848ec5006277/data/data2/current/BP-1933630982-172.17.0.2-1731761461817 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T12:51:43,220 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-16T12:51:43,225 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3e6dcf12{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T12:51:43,226 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@726b5fdd{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T12:51:43,226 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T12:51:43,226 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@131096ea{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T12:51:43,226 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@383b4eca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1df9fbf3-0c5f-03d2-8d05-7680018409c2/hadoop.log.dir/,STOPPED} 2024-11-16T12:51:43,237 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-16T12:51:43,274 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down