2024-12-06 05:39:28,809 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 2024-12-06 05:39:28,849 main DEBUG Took 0.037050 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-06 05:39:28,850 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-06 05:39:28,850 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-06 05:39:28,851 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-06 05:39:28,852 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 05:39:28,863 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-06 05:39:28,877 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 05:39:28,879 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 05:39:28,880 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 05:39:28,880 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 05:39:28,881 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 05:39:28,881 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 05:39:28,883 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 05:39:28,884 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 05:39:28,885 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 05:39:28,885 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 05:39:28,887 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 05:39:28,887 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 05:39:28,888 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 05:39:28,888 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 05:39:28,889 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 05:39:28,890 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 05:39:28,891 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 05:39:28,891 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 05:39:28,892 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 05:39:28,892 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 05:39:28,893 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 05:39:28,893 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 05:39:28,894 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 05:39:28,894 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 05:39:28,895 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 05:39:28,895 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-06 05:39:28,897 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 05:39:28,898 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-06 05:39:28,903 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-06 05:39:28,903 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-06 05:39:28,905 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-06 05:39:28,905 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-06 05:39:28,917 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-06 05:39:28,922 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-06 05:39:28,924 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-06 05:39:28,925 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-06 05:39:28,926 main DEBUG createAppenders(={Console}) 2024-12-06 05:39:28,927 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 initialized 2024-12-06 05:39:28,927 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 2024-12-06 05:39:28,927 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 OK. 2024-12-06 05:39:28,928 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-06 05:39:28,929 main DEBUG OutputStream closed 2024-12-06 05:39:28,929 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-06 05:39:28,930 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-06 05:39:28,930 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@57cf54e1 OK 2024-12-06 05:39:29,027 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-06 05:39:29,029 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-06 05:39:29,031 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-06 05:39:29,033 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-06 05:39:29,033 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-06 05:39:29,034 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-06 05:39:29,034 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-06 05:39:29,035 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-06 05:39:29,035 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-06 05:39:29,035 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-06 05:39:29,036 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-06 05:39:29,036 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-06 05:39:29,036 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-06 05:39:29,037 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-06 05:39:29,037 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-06 05:39:29,037 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-06 05:39:29,038 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-06 05:39:29,039 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-06 05:39:29,042 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-06 05:39:29,042 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6f63b475) with optional ClassLoader: null 2024-12-06 05:39:29,043 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-06 05:39:29,044 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6f63b475] started OK. 2024-12-06T05:39:29,065 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestZooKeeper timeout: 13 mins 2024-12-06 05:39:29,069 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-06 05:39:29,069 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-06T05:39:29,437 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d 2024-12-06T05:39:29,464 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/cluster_05577cd5-3697-aeeb-f42f-dcca3f191df1, deleteOnExit=true 2024-12-06T05:39:29,465 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/test.cache.data in system properties and HBase conf 2024-12-06T05:39:29,465 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/hadoop.tmp.dir in system properties and HBase conf 2024-12-06T05:39:29,466 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/hadoop.log.dir in system properties and HBase conf 2024-12-06T05:39:29,466 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-06T05:39:29,466 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-06T05:39:29,467 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-06T05:39:29,582 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-06T05:39:29,723 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T05:39:29,732 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-06T05:39:29,733 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-06T05:39:29,734 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-06T05:39:29,735 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-06T05:39:29,737 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-06T05:39:29,739 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-06T05:39:29,745 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-06T05:39:29,746 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-06T05:39:29,746 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-06T05:39:29,747 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/nfs.dump.dir in system properties and HBase conf 2024-12-06T05:39:29,747 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/java.io.tmpdir in system properties and HBase conf 2024-12-06T05:39:29,748 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-06T05:39:29,748 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-06T05:39:29,749 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-06T05:39:30,595 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-06T05:39:30,672 INFO [Time-limited test {}] log.Log(170): Logging initialized @2659ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-06T05:39:30,755 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T05:39:30,827 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-06T05:39:30,849 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-06T05:39:30,850 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-06T05:39:30,851 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-06T05:39:30,864 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T05:39:30,867 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2b96f902{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/hadoop.log.dir/,AVAILABLE} 2024-12-06T05:39:30,868 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@640f36cc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-06T05:39:31,078 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2fc737c7{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/java.io.tmpdir/jetty-localhost-46541-hadoop-hdfs-3_4_1-tests_jar-_-any-8114812343118924213/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-06T05:39:31,087 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2be86881{HTTP/1.1, (http/1.1)}{localhost:46541} 2024-12-06T05:39:31,087 INFO [Time-limited test {}] server.Server(415): Started @3075ms 2024-12-06T05:39:31,484 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T05:39:31,491 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-06T05:39:31,492 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-06T05:39:31,492 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-06T05:39:31,492 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-06T05:39:31,493 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@261564d9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/hadoop.log.dir/,AVAILABLE} 2024-12-06T05:39:31,494 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@456b6a03{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-06T05:39:31,604 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5619b125{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/java.io.tmpdir/jetty-localhost-35721-hadoop-hdfs-3_4_1-tests_jar-_-any-1463866033556303329/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T05:39:31,605 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@17dd300f{HTTP/1.1, (http/1.1)}{localhost:35721} 2024-12-06T05:39:31,605 INFO [Time-limited test {}] server.Server(415): Started @3593ms 2024-12-06T05:39:31,676 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-06T05:39:31,793 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T05:39:31,800 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-06T05:39:31,802 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-06T05:39:31,803 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-06T05:39:31,803 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-06T05:39:31,804 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d208e8c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/hadoop.log.dir/,AVAILABLE} 2024-12-06T05:39:31,805 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3be16c75{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-06T05:39:31,929 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3ae9e04a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/java.io.tmpdir/jetty-localhost-34107-hadoop-hdfs-3_4_1-tests_jar-_-any-17088743345398990419/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T05:39:31,930 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1a91a08d{HTTP/1.1, (http/1.1)}{localhost:34107} 2024-12-06T05:39:31,931 INFO [Time-limited test {}] server.Server(415): Started @3918ms 2024-12-06T05:39:31,934 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-06T05:39:32,122 WARN [Thread-94 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/cluster_05577cd5-3697-aeeb-f42f-dcca3f191df1/data/data3/current/BP-170660814-172.17.0.2-1733463570366/current, will proceed with Du for space computation calculation, 2024-12-06T05:39:32,130 WARN [Thread-96 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/cluster_05577cd5-3697-aeeb-f42f-dcca3f191df1/data/data4/current/BP-170660814-172.17.0.2-1733463570366/current, will proceed with Du for space computation calculation, 2024-12-06T05:39:32,131 WARN [Thread-97 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/cluster_05577cd5-3697-aeeb-f42f-dcca3f191df1/data/data1/current/BP-170660814-172.17.0.2-1733463570366/current, will proceed with Du for space computation calculation, 2024-12-06T05:39:32,152 WARN [Thread-99 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/cluster_05577cd5-3697-aeeb-f42f-dcca3f191df1/data/data2/current/BP-170660814-172.17.0.2-1733463570366/current, will proceed with Du for space computation calculation, 2024-12-06T05:39:32,247 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-06T05:39:32,247 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-06T05:39:32,342 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfc2a224e118ddd1b with lease ID 0x737ec09446d0c2bf: Processing first storage report for DS-b2cb688b-dd28-489f-a525-d7192b03d9c9 from datanode DatanodeRegistration(127.0.0.1:43251, datanodeUuid=62a9e372-84d9-4f05-86e0-8fb674bce3fa, infoPort=39027, infoSecurePort=0, ipcPort=38713, storageInfo=lv=-57;cid=testClusterID;nsid=67600604;c=1733463570366) 2024-12-06T05:39:32,344 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfc2a224e118ddd1b with lease ID 0x737ec09446d0c2bf: from storage DS-b2cb688b-dd28-489f-a525-d7192b03d9c9 node DatanodeRegistration(127.0.0.1:43251, datanodeUuid=62a9e372-84d9-4f05-86e0-8fb674bce3fa, infoPort=39027, infoSecurePort=0, ipcPort=38713, storageInfo=lv=-57;cid=testClusterID;nsid=67600604;c=1733463570366), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-06T05:39:32,344 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe19666e89fbc58e with lease ID 0x737ec09446d0c2c0: Processing first storage report for DS-83c8f2e1-f28c-4e6e-b433-9be6a0e4e911 from datanode DatanodeRegistration(127.0.0.1:38675, datanodeUuid=5bec3ff5-1c38-43c4-a60f-974e4f61f1ce, infoPort=41017, infoSecurePort=0, ipcPort=45705, storageInfo=lv=-57;cid=testClusterID;nsid=67600604;c=1733463570366) 2024-12-06T05:39:32,345 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe19666e89fbc58e with lease ID 0x737ec09446d0c2c0: from storage DS-83c8f2e1-f28c-4e6e-b433-9be6a0e4e911 node DatanodeRegistration(127.0.0.1:38675, datanodeUuid=5bec3ff5-1c38-43c4-a60f-974e4f61f1ce, infoPort=41017, infoSecurePort=0, ipcPort=45705, storageInfo=lv=-57;cid=testClusterID;nsid=67600604;c=1733463570366), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-06T05:39:32,345 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfc2a224e118ddd1b with lease ID 0x737ec09446d0c2bf: Processing first storage report for DS-01a6a592-bee1-4cd1-9830-9bb966c9215e from datanode DatanodeRegistration(127.0.0.1:43251, datanodeUuid=62a9e372-84d9-4f05-86e0-8fb674bce3fa, infoPort=39027, infoSecurePort=0, ipcPort=38713, storageInfo=lv=-57;cid=testClusterID;nsid=67600604;c=1733463570366) 2024-12-06T05:39:32,345 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfc2a224e118ddd1b with lease ID 0x737ec09446d0c2bf: from storage DS-01a6a592-bee1-4cd1-9830-9bb966c9215e node DatanodeRegistration(127.0.0.1:43251, datanodeUuid=62a9e372-84d9-4f05-86e0-8fb674bce3fa, infoPort=39027, infoSecurePort=0, ipcPort=38713, storageInfo=lv=-57;cid=testClusterID;nsid=67600604;c=1733463570366), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-06T05:39:32,345 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe19666e89fbc58e with lease ID 0x737ec09446d0c2c0: Processing first storage report for DS-d319434e-e45f-4b06-8498-3269be5d7781 from datanode DatanodeRegistration(127.0.0.1:38675, datanodeUuid=5bec3ff5-1c38-43c4-a60f-974e4f61f1ce, infoPort=41017, infoSecurePort=0, ipcPort=45705, storageInfo=lv=-57;cid=testClusterID;nsid=67600604;c=1733463570366) 2024-12-06T05:39:32,346 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe19666e89fbc58e with lease ID 0x737ec09446d0c2c0: from storage DS-d319434e-e45f-4b06-8498-3269be5d7781 node DatanodeRegistration(127.0.0.1:38675, datanodeUuid=5bec3ff5-1c38-43c4-a60f-974e4f61f1ce, infoPort=41017, infoSecurePort=0, ipcPort=45705, storageInfo=lv=-57;cid=testClusterID;nsid=67600604;c=1733463570366), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-06T05:39:32,409 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d 2024-12-06T05:39:32,507 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/cluster_05577cd5-3697-aeeb-f42f-dcca3f191df1/zookeeper_0, clientPort=58086, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/cluster_05577cd5-3697-aeeb-f42f-dcca3f191df1/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/cluster_05577cd5-3697-aeeb-f42f-dcca3f191df1/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-06T05:39:32,519 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58086 2024-12-06T05:39:32,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/test.cache.data in system properties and HBase conf 2024-12-06T05:39:32,522 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/hadoop.tmp.dir in system properties and HBase conf 2024-12-06T05:39:32,522 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/hadoop.log.dir in system properties and HBase conf 2024-12-06T05:39:32,522 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-06T05:39:32,522 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-06T05:39:32,522 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-06T05:39:32,542 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:32,546 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:32,557 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-06T05:39:32,558 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-06T05:39:32,558 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-06T05:39:32,558 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-06T05:39:32,558 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-06T05:39:32,559 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-06T05:39:32,559 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-06T05:39:32,559 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-06T05:39:32,559 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-06T05:39:32,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/nfs.dump.dir in system properties and HBase conf 2024-12-06T05:39:32,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/java.io.tmpdir in system properties and HBase conf 2024-12-06T05:39:32,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-06T05:39:32,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-06T05:39:32,561 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-06T05:39:32,654 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T05:39:32,664 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-06T05:39:32,679 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-06T05:39:32,679 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-06T05:39:32,680 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-06T05:39:32,681 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T05:39:32,682 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5945132b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/hadoop.log.dir/,AVAILABLE} 2024-12-06T05:39:32,683 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@246c61b7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-06T05:39:32,821 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@8f70d69{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/java.io.tmpdir/jetty-localhost-32925-hadoop-hdfs-3_4_1-tests_jar-_-any-13784399797765421279/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-06T05:39:32,822 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@62f5f276{HTTP/1.1, (http/1.1)}{localhost:32925} 2024-12-06T05:39:32,823 INFO [Time-limited test {}] server.Server(415): Started @4810ms 2024-12-06T05:39:32,996 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T05:39:33,003 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-06T05:39:33,009 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-06T05:39:33,010 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-06T05:39:33,010 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-06T05:39:33,012 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5e03d9d9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/hadoop.log.dir/,AVAILABLE} 2024-12-06T05:39:33,013 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@599e8739{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-06T05:39:33,147 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4a8ffa8b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/java.io.tmpdir/jetty-localhost-44791-hadoop-hdfs-3_4_1-tests_jar-_-any-11624640090388584350/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T05:39:33,148 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@aedc167{HTTP/1.1, (http/1.1)}{localhost:44791} 2024-12-06T05:39:33,148 INFO [Time-limited test {}] server.Server(415): Started @5136ms 2024-12-06T05:39:33,151 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-06T05:39:33,208 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T05:39:33,213 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-06T05:39:33,217 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-06T05:39:33,218 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-06T05:39:33,218 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-06T05:39:33,219 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@204c24bc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/hadoop.log.dir/,AVAILABLE} 2024-12-06T05:39:33,219 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ac046fe{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-06T05:39:33,232 WARN [Thread-190 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/cluster_05577cd5-3697-aeeb-f42f-dcca3f191df1/data/data2/current/BP-1951228416-172.17.0.2-1733463572602/current, will proceed with Du for space computation calculation, 2024-12-06T05:39:33,232 WARN [Thread-189 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/cluster_05577cd5-3697-aeeb-f42f-dcca3f191df1/data/data1/current/BP-1951228416-172.17.0.2-1733463572602/current, will proceed with Du for space computation calculation, 2024-12-06T05:39:33,258 WARN [Thread-168 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-06T05:39:33,264 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc5d070b17f6d1fbe with lease ID 0x7c3b60ef54bac4fe: Processing first storage report for DS-4485eaa6-367b-4084-97e1-b834fb8d8596 from datanode DatanodeRegistration(127.0.0.1:39561, datanodeUuid=ffe5ff56-7611-4f34-97e0-03f4d3941f40, infoPort=43559, infoSecurePort=0, ipcPort=42031, storageInfo=lv=-57;cid=testClusterID;nsid=2070955352;c=1733463572602) 2024-12-06T05:39:33,264 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc5d070b17f6d1fbe with lease ID 0x7c3b60ef54bac4fe: from storage DS-4485eaa6-367b-4084-97e1-b834fb8d8596 node DatanodeRegistration(127.0.0.1:39561, datanodeUuid=ffe5ff56-7611-4f34-97e0-03f4d3941f40, infoPort=43559, infoSecurePort=0, ipcPort=42031, storageInfo=lv=-57;cid=testClusterID;nsid=2070955352;c=1733463572602), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-06T05:39:33,264 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc5d070b17f6d1fbe with lease ID 0x7c3b60ef54bac4fe: Processing first storage report for DS-17c85513-a4a4-4fe0-a6b4-4c3fe8118b86 from datanode DatanodeRegistration(127.0.0.1:39561, datanodeUuid=ffe5ff56-7611-4f34-97e0-03f4d3941f40, infoPort=43559, infoSecurePort=0, ipcPort=42031, storageInfo=lv=-57;cid=testClusterID;nsid=2070955352;c=1733463572602) 2024-12-06T05:39:33,264 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc5d070b17f6d1fbe with lease ID 0x7c3b60ef54bac4fe: from storage DS-17c85513-a4a4-4fe0-a6b4-4c3fe8118b86 node DatanodeRegistration(127.0.0.1:39561, datanodeUuid=ffe5ff56-7611-4f34-97e0-03f4d3941f40, infoPort=43559, infoSecurePort=0, ipcPort=42031, storageInfo=lv=-57;cid=testClusterID;nsid=2070955352;c=1733463572602), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-06T05:39:33,320 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2dc9b531{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/java.io.tmpdir/jetty-localhost-36737-hadoop-hdfs-3_4_1-tests_jar-_-any-11104349641854896240/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T05:39:33,320 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43302dd2{HTTP/1.1, (http/1.1)}{localhost:36737} 2024-12-06T05:39:33,320 INFO [Time-limited test {}] server.Server(415): Started @5308ms 2024-12-06T05:39:33,322 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-06T05:39:33,389 WARN [Thread-215 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/cluster_05577cd5-3697-aeeb-f42f-dcca3f191df1/data/data3/current/BP-1951228416-172.17.0.2-1733463572602/current, will proceed with Du for space computation calculation, 2024-12-06T05:39:33,390 WARN [Thread-216 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/cluster_05577cd5-3697-aeeb-f42f-dcca3f191df1/data/data4/current/BP-1951228416-172.17.0.2-1733463572602/current, will proceed with Du for space computation calculation, 2024-12-06T05:39:33,412 WARN [Thread-204 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-06T05:39:33,416 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x43fc2a6304f74b95 with lease ID 0x7c3b60ef54bac4ff: Processing first storage report for DS-80ddb16c-ec66-4e39-a44c-0a80fa143251 from datanode DatanodeRegistration(127.0.0.1:40949, datanodeUuid=e1e65c20-ad02-4f59-b331-e60f89cea478, infoPort=44855, infoSecurePort=0, ipcPort=34047, storageInfo=lv=-57;cid=testClusterID;nsid=2070955352;c=1733463572602) 2024-12-06T05:39:33,416 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x43fc2a6304f74b95 with lease ID 0x7c3b60ef54bac4ff: from storage DS-80ddb16c-ec66-4e39-a44c-0a80fa143251 node DatanodeRegistration(127.0.0.1:40949, datanodeUuid=e1e65c20-ad02-4f59-b331-e60f89cea478, infoPort=44855, infoSecurePort=0, ipcPort=34047, storageInfo=lv=-57;cid=testClusterID;nsid=2070955352;c=1733463572602), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-06T05:39:33,417 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x43fc2a6304f74b95 with lease ID 0x7c3b60ef54bac4ff: Processing first storage report for DS-aae85e5f-7382-4620-a5e9-f386408d3f82 from datanode DatanodeRegistration(127.0.0.1:40949, datanodeUuid=e1e65c20-ad02-4f59-b331-e60f89cea478, infoPort=44855, infoSecurePort=0, ipcPort=34047, storageInfo=lv=-57;cid=testClusterID;nsid=2070955352;c=1733463572602) 2024-12-06T05:39:33,417 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x43fc2a6304f74b95 with lease ID 0x7c3b60ef54bac4ff: from storage DS-aae85e5f-7382-4620-a5e9-f386408d3f82 node DatanodeRegistration(127.0.0.1:40949, datanodeUuid=e1e65c20-ad02-4f59-b331-e60f89cea478, infoPort=44855, infoSecurePort=0, ipcPort=34047, storageInfo=lv=-57;cid=testClusterID;nsid=2070955352;c=1733463572602), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-06T05:39:33,452 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d 2024-12-06T05:39:33,465 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestZooKeeper#testRegionServerSessionExpired Thread=236, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=312, ProcessCount=11, AvailableMemoryMB=5579 2024-12-06T05:39:33,467 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:33,469 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:33,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741825_1001 (size=7) 2024-12-06T05:39:33,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741825_1001 (size=7) 2024-12-06T05:39:34,084 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 with version=8 2024-12-06T05:39:34,084 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/hbase-staging 2024-12-06T05:39:34,175 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-06T05:39:34,408 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b543b351fe95:0 server-side Connection retries=45 2024-12-06T05:39:34,418 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:34,418 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:34,423 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T05:39:34,423 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:34,423 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T05:39:34,544 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-06T05:39:34,595 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-06T05:39:34,605 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-06T05:39:34,609 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T05:39:34,633 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1532 (auto-detected) 2024-12-06T05:39:34,634 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-06T05:39:34,649 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42763 2024-12-06T05:39:34,666 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42763 connecting to ZooKeeper ensemble=127.0.0.1:58086 2024-12-06T05:39:34,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:427630x0, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:39:34,693 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42763-0x100ae0a5c730000 connected 2024-12-06T05:39:34,716 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:34,717 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:34,725 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42763-0x100ae0a5c730000, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T05:39:34,729 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279, hbase.cluster.distributed=false 2024-12-06T05:39:34,750 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42763-0x100ae0a5c730000, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T05:39:34,755 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42763 2024-12-06T05:39:34,756 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42763 2024-12-06T05:39:34,757 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42763 2024-12-06T05:39:34,757 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42763 2024-12-06T05:39:34,758 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42763 2024-12-06T05:39:34,762 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b543b351fe95:0 server-side Connection retries=45 2024-12-06T05:39:34,762 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:34,762 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:34,762 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T05:39:34,763 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:34,763 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T05:39:34,763 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-06T05:39:34,763 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T05:39:34,765 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44973 2024-12-06T05:39:34,766 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44973 connecting to ZooKeeper ensemble=127.0.0.1:58086 2024-12-06T05:39:34,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:449730x0, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:39:34,772 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44973-0x100ae0a5c730001 connected 2024-12-06T05:39:34,778 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:34,779 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:34,782 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44973-0x100ae0a5c730001, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T05:39:34,783 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279, hbase.cluster.distributed=false 2024-12-06T05:39:34,786 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44973-0x100ae0a5c730001, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T05:39:34,786 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44973 2024-12-06T05:39:34,787 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44973 2024-12-06T05:39:34,787 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44973 2024-12-06T05:39:34,788 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44973 2024-12-06T05:39:34,788 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44973 2024-12-06T05:39:34,888 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b543b351fe95:0 server-side Connection retries=45 2024-12-06T05:39:34,890 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:34,890 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:34,890 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T05:39:34,890 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:34,890 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T05:39:34,894 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T05:39:34,897 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T05:39:34,898 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42549 2024-12-06T05:39:34,900 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42549 connecting to ZooKeeper ensemble=127.0.0.1:58086 2024-12-06T05:39:34,901 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:34,904 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:34,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:425490x0, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:39:34,910 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42549-0x100ae0a5c730002 connected 2024-12-06T05:39:34,911 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42549-0x100ae0a5c730002, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T05:39:34,916 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T05:39:34,925 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T05:39:34,928 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42549-0x100ae0a5c730002, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:39:34,934 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42549-0x100ae0a5c730002, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T05:39:34,935 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42549 2024-12-06T05:39:34,936 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42549 2024-12-06T05:39:34,936 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42549 2024-12-06T05:39:34,937 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42549 2024-12-06T05:39:34,937 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42549 2024-12-06T05:39:34,986 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b543b351fe95:0 server-side Connection retries=45 2024-12-06T05:39:34,986 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:34,986 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:34,986 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T05:39:34,986 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:34,986 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T05:39:34,986 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T05:39:34,987 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T05:39:34,988 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45699 2024-12-06T05:39:34,989 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45699 connecting to ZooKeeper ensemble=127.0.0.1:58086 2024-12-06T05:39:34,991 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:34,994 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:34,999 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:456990x0, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T05:39:35,000 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T05:39:35,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:456990x0, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:39:35,003 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45699-0x100ae0a5c730003 connected 2024-12-06T05:39:35,003 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T05:39:35,004 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45699-0x100ae0a5c730003, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:39:35,006 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45699-0x100ae0a5c730003, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T05:39:35,008 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45699 2024-12-06T05:39:35,009 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45699 2024-12-06T05:39:35,009 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45699 2024-12-06T05:39:35,010 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45699 2024-12-06T05:39:35,010 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45699 2024-12-06T05:39:35,030 DEBUG [M:0;b543b351fe95:42763 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b543b351fe95:42763 2024-12-06T05:39:35,030 DEBUG [M:1;b543b351fe95:44973 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:1;b543b351fe95:44973 2024-12-06T05:39:35,031 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b543b351fe95,42763,1733463574228 2024-12-06T05:39:35,031 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b543b351fe95,44973,1733463574761 2024-12-06T05:39:35,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44973-0x100ae0a5c730001, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:35,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x100ae0a5c730000, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:35,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45699-0x100ae0a5c730003, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:35,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42549-0x100ae0a5c730002, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:35,037 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44973-0x100ae0a5c730001, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b543b351fe95,44973,1733463574761 2024-12-06T05:39:35,039 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42763-0x100ae0a5c730000, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b543b351fe95,42763,1733463574228 2024-12-06T05:39:35,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42549-0x100ae0a5c730002, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:35,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x100ae0a5c730000, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:35,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44973-0x100ae0a5c730001, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:35,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42549-0x100ae0a5c730002, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:35,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45699-0x100ae0a5c730003, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:35,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45699-0x100ae0a5c730003, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:35,059 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44973-0x100ae0a5c730001, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T05:39:35,059 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42763-0x100ae0a5c730000, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T05:39:35,060 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ActiveMasterManager(296): Another master is the active master, b543b351fe95,44973,1733463574761; waiting to become the next active master 2024-12-06T05:39:35,060 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b543b351fe95,44973,1733463574761 from backup master directory 2024-12-06T05:39:35,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44973-0x100ae0a5c730001, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b543b351fe95,44973,1733463574761 2024-12-06T05:39:35,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45699-0x100ae0a5c730003, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:35,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44973-0x100ae0a5c730001, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:35,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42549-0x100ae0a5c730002, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:35,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x100ae0a5c730000, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:35,064 WARN [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T05:39:35,064 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b543b351fe95,44973,1733463574761 2024-12-06T05:39:35,066 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-06T05:39:35,067 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-06T05:39:35,118 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/hbase.id] with ID: 31dab96b-d1b9-4395-a5ed-7d1e827cdbcc 2024-12-06T05:39:35,118 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/.tmp/hbase.id 2024-12-06T05:39:35,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741826_1002 (size=42) 2024-12-06T05:39:35,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741826_1002 (size=42) 2024-12-06T05:39:35,532 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/.tmp/hbase.id]:[hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/hbase.id] 2024-12-06T05:39:35,577 INFO [master/b543b351fe95:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:35,582 INFO [master/b543b351fe95:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-06T05:39:35,600 INFO [master/b543b351fe95:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-12-06T05:39:35,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44973-0x100ae0a5c730001, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:35,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45699-0x100ae0a5c730003, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:35,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x100ae0a5c730000, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:35,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42549-0x100ae0a5c730002, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:35,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741827_1003 (size=196) 2024-12-06T05:39:35,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741827_1003 (size=196) 2024-12-06T05:39:35,633 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T05:39:35,635 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-06T05:39:35,648 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:39:35,652 INFO [master/b543b351fe95:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:39:35,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741828_1004 (size=1189) 2024-12-06T05:39:35,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741828_1004 (size=1189) 2024-12-06T05:39:35,694 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store 2024-12-06T05:39:35,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741829_1005 (size=34) 2024-12-06T05:39:35,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741829_1005 (size=34) 2024-12-06T05:39:35,716 INFO [master/b543b351fe95:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-06T05:39:35,718 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:35,719 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T05:39:35,719 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:35,720 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:35,721 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T05:39:35,721 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:35,721 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:35,722 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733463575719Disabling compacts and flushes for region at 1733463575719Disabling writes for close at 1733463575721 (+2 ms)Writing region close event to WAL at 1733463575721Closed at 1733463575721 2024-12-06T05:39:35,725 WARN [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/.initializing 2024-12-06T05:39:35,725 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,44973,1733463574761 2024-12-06T05:39:35,732 INFO [master/b543b351fe95:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:39:35,746 INFO [master/b543b351fe95:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b543b351fe95%2C44973%2C1733463574761, suffix=, logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,44973,1733463574761, archiveDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/oldWALs, maxLogs=10 2024-12-06T05:39:35,765 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,44973,1733463574761/b543b351fe95%2C44973%2C1733463574761.1733463575750, exclude list is [], retry=0 2024-12-06T05:39:35,780 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39561,DS-4485eaa6-367b-4084-97e1-b834fb8d8596,DISK] 2024-12-06T05:39:35,780 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-80ddb16c-ec66-4e39-a44c-0a80fa143251,DISK] 2024-12-06T05:39:35,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-06T05:39:35,814 INFO [master/b543b351fe95:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,44973,1733463574761/b543b351fe95%2C44973%2C1733463574761.1733463575750 2024-12-06T05:39:35,815 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43559:43559),(127.0.0.1/127.0.0.1:44855:44855)] 2024-12-06T05:39:35,816 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:39:35,816 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:35,819 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:35,820 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:35,853 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:35,879 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-06T05:39:35,884 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:35,887 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:35,887 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:35,891 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-06T05:39:35,891 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:35,892 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:39:35,892 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:35,896 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-06T05:39:35,896 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:35,898 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:39:35,898 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:35,902 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-06T05:39:35,902 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:35,904 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:39:35,904 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:35,908 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:35,910 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:35,917 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:35,917 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:35,922 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T05:39:35,926 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:35,932 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:39:35,933 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68924471, jitterRate=0.02705465257167816}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T05:39:35,939 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733463575832Initializing all the Stores at 1733463575834 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463575835 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463575835Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463575835Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463575835Cleaning up temporary data from old regions at 1733463575917 (+82 ms)Region opened successfully at 1733463575939 (+22 ms) 2024-12-06T05:39:35,940 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-06T05:39:35,979 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d65006, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b543b351fe95/172.17.0.2:0 2024-12-06T05:39:36,018 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-06T05:39:36,033 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-06T05:39:36,033 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-06T05:39:36,037 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-06T05:39:36,039 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-06T05:39:36,046 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-12-06T05:39:36,046 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-06T05:39:36,076 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-06T05:39:36,087 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44973-0x100ae0a5c730001, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-06T05:39:36,088 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-06T05:39:36,091 INFO [master/b543b351fe95:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-06T05:39:36,093 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44973-0x100ae0a5c730001, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-06T05:39:36,095 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-06T05:39:36,097 INFO [master/b543b351fe95:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-06T05:39:36,101 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44973-0x100ae0a5c730001, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-06T05:39:36,103 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-06T05:39:36,105 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44973-0x100ae0a5c730001, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-06T05:39:36,106 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-06T05:39:36,125 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44973-0x100ae0a5c730001, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-06T05:39:36,126 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-06T05:39:36,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45699-0x100ae0a5c730003, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T05:39:36,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42549-0x100ae0a5c730002, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T05:39:36,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45699-0x100ae0a5c730003, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:36,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42549-0x100ae0a5c730002, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:36,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x100ae0a5c730000, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T05:39:36,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44973-0x100ae0a5c730001, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T05:39:36,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44973-0x100ae0a5c730001, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:36,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x100ae0a5c730000, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:36,132 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=b543b351fe95,44973,1733463574761, sessionid=0x100ae0a5c730001, setting cluster-up flag (Was=false) 2024-12-06T05:39:36,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42549-0x100ae0a5c730002, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:36,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x100ae0a5c730000, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:36,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44973-0x100ae0a5c730001, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:36,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45699-0x100ae0a5c730003, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:36,152 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-06T05:39:36,154 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b543b351fe95,44973,1733463574761 2024-12-06T05:39:36,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42549-0x100ae0a5c730002, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:36,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45699-0x100ae0a5c730003, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:36,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44973-0x100ae0a5c730001, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:36,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x100ae0a5c730000, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:36,165 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-06T05:39:36,167 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b543b351fe95,44973,1733463574761 2024-12-06T05:39:36,175 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-06T05:39:36,214 INFO [RS:1;b543b351fe95:45699 {}] regionserver.HRegionServer(746): ClusterId : 31dab96b-d1b9-4395-a5ed-7d1e827cdbcc 2024-12-06T05:39:36,214 INFO [RS:0;b543b351fe95:42549 {}] regionserver.HRegionServer(746): ClusterId : 31dab96b-d1b9-4395-a5ed-7d1e827cdbcc 2024-12-06T05:39:36,216 DEBUG [RS:0;b543b351fe95:42549 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T05:39:36,216 DEBUG [RS:1;b543b351fe95:45699 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T05:39:36,220 DEBUG [RS:1;b543b351fe95:45699 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T05:39:36,220 DEBUG [RS:0;b543b351fe95:42549 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T05:39:36,220 DEBUG [RS:1;b543b351fe95:45699 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T05:39:36,220 DEBUG [RS:0;b543b351fe95:42549 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T05:39:36,223 DEBUG [RS:0;b543b351fe95:42549 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T05:39:36,223 DEBUG [RS:1;b543b351fe95:45699 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T05:39:36,223 DEBUG [RS:0;b543b351fe95:42549 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66f089ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b543b351fe95/172.17.0.2:0 2024-12-06T05:39:36,223 DEBUG [RS:1;b543b351fe95:45699 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4554955f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b543b351fe95/172.17.0.2:0 2024-12-06T05:39:36,237 DEBUG [RS:0;b543b351fe95:42549 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b543b351fe95:42549 2024-12-06T05:39:36,241 INFO [RS:0;b543b351fe95:42549 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T05:39:36,241 INFO [RS:0;b543b351fe95:42549 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T05:39:36,241 DEBUG [RS:0;b543b351fe95:42549 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T05:39:36,244 INFO [RS:0;b543b351fe95:42549 {}] regionserver.HRegionServer(2659): reportForDuty to master=b543b351fe95,44973,1733463574761 with port=42549, startcode=1733463574853 2024-12-06T05:39:36,248 DEBUG [RS:1;b543b351fe95:45699 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;b543b351fe95:45699 2024-12-06T05:39:36,248 INFO [RS:1;b543b351fe95:45699 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T05:39:36,248 INFO [RS:1;b543b351fe95:45699 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T05:39:36,248 DEBUG [RS:1;b543b351fe95:45699 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T05:39:36,249 INFO [RS:1;b543b351fe95:45699 {}] regionserver.HRegionServer(2659): reportForDuty to master=b543b351fe95,44973,1733463574761 with port=45699, startcode=1733463574985 2024-12-06T05:39:36,257 DEBUG [RS:1;b543b351fe95:45699 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T05:39:36,257 DEBUG [RS:0;b543b351fe95:42549 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T05:39:36,272 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-06T05:39:36,285 INFO [master/b543b351fe95:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-06T05:39:36,294 INFO [HMaster-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46555, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T05:39:36,294 INFO [HMaster-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41695, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T05:39:36,292 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b543b351fe95,44973,1733463574761 Number of backup masters: 1 b543b351fe95,42763,1733463574228 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-06T05:39:36,300 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44973 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:36,301 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:39:36,301 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:39:36,301 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:39:36,301 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:39:36,301 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b543b351fe95:0, corePoolSize=10, maxPoolSize=10 2024-12-06T05:39:36,302 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:36,302 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b543b351fe95:0, corePoolSize=2, maxPoolSize=2 2024-12-06T05:39:36,302 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:36,305 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44973 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:36,308 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T05:39:36,309 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-06T05:39:36,315 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733463606315 2024-12-06T05:39:36,316 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:36,317 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-06T05:39:36,317 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-06T05:39:36,319 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-06T05:39:36,323 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-06T05:39:36,323 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-06T05:39:36,324 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-06T05:39:36,324 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-06T05:39:36,330 DEBUG [RS:0;b543b351fe95:42549 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-06T05:39:36,330 DEBUG [RS:1;b543b351fe95:45699 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-06T05:39:36,330 WARN [RS:0;b543b351fe95:42549 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-06T05:39:36,330 WARN [RS:1;b543b351fe95:45699 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-06T05:39:36,329 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:36,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741831_1007 (size=1321) 2024-12-06T05:39:36,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741831_1007 (size=1321) 2024-12-06T05:39:36,341 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-06T05:39:36,342 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:39:36,345 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-06T05:39:36,347 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-06T05:39:36,348 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-06T05:39:36,351 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-06T05:39:36,351 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-06T05:39:36,354 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.large.0-1733463576353,5,FailOnTimeoutGroup] 2024-12-06T05:39:36,354 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.small.0-1733463576354,5,FailOnTimeoutGroup] 2024-12-06T05:39:36,355 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:36,355 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-06T05:39:36,356 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:36,357 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:36,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741832_1008 (size=32) 2024-12-06T05:39:36,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741832_1008 (size=32) 2024-12-06T05:39:36,373 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:36,377 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-06T05:39:36,380 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T05:39:36,380 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:36,382 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:36,382 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-06T05:39:36,385 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T05:39:36,385 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:36,387 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:36,387 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-06T05:39:36,390 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T05:39:36,391 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:36,392 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:36,392 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-06T05:39:36,395 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T05:39:36,395 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:36,396 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:36,396 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T05:39:36,397 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/hbase/meta/1588230740 2024-12-06T05:39:36,398 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/hbase/meta/1588230740 2024-12-06T05:39:36,401 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T05:39:36,401 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T05:39:36,402 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T05:39:36,405 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T05:39:36,409 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:39:36,410 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65090250, jitterRate=-0.030079692602157593}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T05:39:36,413 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733463576373Initializing all the Stores at 1733463576376 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463576376Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463576377 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463576377Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463576377Cleaning up temporary data from old regions at 1733463576402 (+25 ms)Region opened successfully at 1733463576413 (+11 ms) 2024-12-06T05:39:36,413 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T05:39:36,413 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T05:39:36,413 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T05:39:36,414 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T05:39:36,414 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T05:39:36,415 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T05:39:36,415 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733463576413Disabling compacts and flushes for region at 1733463576413Disabling writes for close at 1733463576414 (+1 ms)Writing region close event to WAL at 1733463576415 (+1 ms)Closed at 1733463576415 2024-12-06T05:39:36,422 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T05:39:36,422 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-06T05:39:36,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-06T05:39:36,431 INFO [RS:0;b543b351fe95:42549 {}] regionserver.HRegionServer(2659): reportForDuty to master=b543b351fe95,44973,1733463574761 with port=42549, startcode=1733463574853 2024-12-06T05:39:36,431 INFO [RS:1;b543b351fe95:45699 {}] regionserver.HRegionServer(2659): reportForDuty to master=b543b351fe95,44973,1733463574761 with port=45699, startcode=1733463574985 2024-12-06T05:39:36,433 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44973 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b543b351fe95,42549,1733463574853 2024-12-06T05:39:36,436 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44973 {}] master.ServerManager(517): Registering regionserver=b543b351fe95,42549,1733463574853 2024-12-06T05:39:36,441 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T05:39:36,444 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-06T05:39:36,446 DEBUG [RS:0;b543b351fe95:42549 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:39:36,446 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44973 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b543b351fe95,45699,1733463574985 2024-12-06T05:39:36,447 DEBUG [RS:0;b543b351fe95:42549 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34453 2024-12-06T05:39:36,447 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44973 {}] master.ServerManager(517): Registering regionserver=b543b351fe95,45699,1733463574985 2024-12-06T05:39:36,447 DEBUG [RS:0;b543b351fe95:42549 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T05:39:36,450 DEBUG [RS:1;b543b351fe95:45699 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:39:36,451 DEBUG [RS:1;b543b351fe95:45699 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34453 2024-12-06T05:39:36,451 DEBUG [RS:1;b543b351fe95:45699 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T05:39:36,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x100ae0a5c730000, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T05:39:36,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44973-0x100ae0a5c730001, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T05:39:36,454 DEBUG [RS:0;b543b351fe95:42549 {}] zookeeper.ZKUtil(111): regionserver:42549-0x100ae0a5c730002, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b543b351fe95,42549,1733463574853 2024-12-06T05:39:36,454 DEBUG [RS:1;b543b351fe95:45699 {}] zookeeper.ZKUtil(111): regionserver:45699-0x100ae0a5c730003, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b543b351fe95,45699,1733463574985 2024-12-06T05:39:36,454 WARN [RS:0;b543b351fe95:42549 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T05:39:36,454 WARN [RS:1;b543b351fe95:45699 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T05:39:36,455 INFO [RS:0;b543b351fe95:42549 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:39:36,455 INFO [RS:1;b543b351fe95:45699 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:39:36,455 DEBUG [RS:0;b543b351fe95:42549 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,42549,1733463574853 2024-12-06T05:39:36,455 DEBUG [RS:1;b543b351fe95:45699 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,45699,1733463574985 2024-12-06T05:39:36,459 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b543b351fe95,42549,1733463574853] 2024-12-06T05:39:36,460 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b543b351fe95,45699,1733463574985] 2024-12-06T05:39:36,486 INFO [RS:1;b543b351fe95:45699 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T05:39:36,486 INFO [RS:0;b543b351fe95:42549 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T05:39:36,499 INFO [RS:1;b543b351fe95:45699 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T05:39:36,500 INFO [RS:0;b543b351fe95:42549 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T05:39:36,505 INFO [RS:0;b543b351fe95:42549 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T05:39:36,505 INFO [RS:1;b543b351fe95:45699 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T05:39:36,506 INFO [RS:0;b543b351fe95:42549 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:36,506 INFO [RS:1;b543b351fe95:45699 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:36,507 INFO [RS:0;b543b351fe95:42549 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T05:39:36,507 INFO [RS:1;b543b351fe95:45699 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T05:39:36,513 INFO [RS:0;b543b351fe95:42549 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T05:39:36,513 INFO [RS:1;b543b351fe95:45699 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T05:39:36,515 INFO [RS:1;b543b351fe95:45699 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:36,515 INFO [RS:0;b543b351fe95:42549 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:36,515 DEBUG [RS:1;b543b351fe95:45699 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:36,515 DEBUG [RS:0;b543b351fe95:42549 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:36,515 DEBUG [RS:0;b543b351fe95:42549 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:36,515 DEBUG [RS:1;b543b351fe95:45699 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:36,515 DEBUG [RS:0;b543b351fe95:42549 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:36,515 DEBUG [RS:1;b543b351fe95:45699 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:36,516 DEBUG [RS:0;b543b351fe95:42549 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:36,516 DEBUG [RS:1;b543b351fe95:45699 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:36,516 DEBUG [RS:1;b543b351fe95:45699 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:36,516 DEBUG [RS:0;b543b351fe95:42549 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:36,516 DEBUG [RS:0;b543b351fe95:42549 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0, corePoolSize=2, maxPoolSize=2 2024-12-06T05:39:36,516 DEBUG [RS:1;b543b351fe95:45699 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0, corePoolSize=2, maxPoolSize=2 2024-12-06T05:39:36,516 DEBUG [RS:0;b543b351fe95:42549 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:36,516 DEBUG [RS:1;b543b351fe95:45699 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:36,516 DEBUG [RS:0;b543b351fe95:42549 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:36,516 DEBUG [RS:1;b543b351fe95:45699 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:36,516 DEBUG [RS:1;b543b351fe95:45699 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:36,516 DEBUG [RS:0;b543b351fe95:42549 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:36,516 DEBUG [RS:0;b543b351fe95:42549 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:36,516 DEBUG [RS:1;b543b351fe95:45699 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:36,517 DEBUG [RS:1;b543b351fe95:45699 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:36,517 DEBUG [RS:0;b543b351fe95:42549 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:36,517 DEBUG [RS:1;b543b351fe95:45699 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:36,517 DEBUG [RS:0;b543b351fe95:42549 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:36,517 DEBUG [RS:1;b543b351fe95:45699 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b543b351fe95:0, corePoolSize=3, maxPoolSize=3 2024-12-06T05:39:36,517 DEBUG [RS:0;b543b351fe95:42549 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b543b351fe95:0, corePoolSize=3, maxPoolSize=3 2024-12-06T05:39:36,517 DEBUG [RS:0;b543b351fe95:42549 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b543b351fe95:0, corePoolSize=3, maxPoolSize=3 2024-12-06T05:39:36,517 DEBUG [RS:1;b543b351fe95:45699 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b543b351fe95:0, corePoolSize=3, maxPoolSize=3 2024-12-06T05:39:36,523 INFO [RS:0;b543b351fe95:42549 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:36,523 INFO [RS:0;b543b351fe95:42549 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:36,523 INFO [RS:0;b543b351fe95:42549 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:36,524 INFO [RS:0;b543b351fe95:42549 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:36,524 INFO [RS:0;b543b351fe95:42549 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:36,524 INFO [RS:0;b543b351fe95:42549 {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,42549,1733463574853-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T05:39:36,533 INFO [RS:1;b543b351fe95:45699 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:36,533 INFO [RS:1;b543b351fe95:45699 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:36,533 INFO [RS:1;b543b351fe95:45699 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:36,534 INFO [RS:1;b543b351fe95:45699 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:36,534 INFO [RS:1;b543b351fe95:45699 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:36,534 INFO [RS:1;b543b351fe95:45699 {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,45699,1733463574985-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T05:39:36,555 INFO [RS:0;b543b351fe95:42549 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T05:39:36,556 INFO [RS:1;b543b351fe95:45699 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T05:39:36,557 INFO [RS:0;b543b351fe95:42549 {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,42549,1733463574853-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:36,557 INFO [RS:1;b543b351fe95:45699 {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,45699,1733463574985-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:36,557 INFO [RS:0;b543b351fe95:42549 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:36,557 INFO [RS:1;b543b351fe95:45699 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:36,557 INFO [RS:0;b543b351fe95:42549 {}] regionserver.Replication(171): b543b351fe95,42549,1733463574853 started 2024-12-06T05:39:36,557 INFO [RS:1;b543b351fe95:45699 {}] regionserver.Replication(171): b543b351fe95,45699,1733463574985 started 2024-12-06T05:39:36,580 INFO [RS:0;b543b351fe95:42549 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:36,581 INFO [RS:0;b543b351fe95:42549 {}] regionserver.HRegionServer(1482): Serving as b543b351fe95,42549,1733463574853, RpcServer on b543b351fe95/172.17.0.2:42549, sessionid=0x100ae0a5c730002 2024-12-06T05:39:36,582 DEBUG [RS:0;b543b351fe95:42549 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T05:39:36,582 DEBUG [RS:0;b543b351fe95:42549 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b543b351fe95,42549,1733463574853 2024-12-06T05:39:36,582 DEBUG [RS:0;b543b351fe95:42549 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b543b351fe95,42549,1733463574853' 2024-12-06T05:39:36,582 DEBUG [RS:0;b543b351fe95:42549 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T05:39:36,583 DEBUG [RS:0;b543b351fe95:42549 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T05:39:36,584 DEBUG [RS:0;b543b351fe95:42549 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T05:39:36,585 DEBUG [RS:0;b543b351fe95:42549 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T05:39:36,585 DEBUG [RS:0;b543b351fe95:42549 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b543b351fe95,42549,1733463574853 2024-12-06T05:39:36,585 DEBUG [RS:0;b543b351fe95:42549 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b543b351fe95,42549,1733463574853' 2024-12-06T05:39:36,585 DEBUG [RS:0;b543b351fe95:42549 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T05:39:36,586 DEBUG [RS:0;b543b351fe95:42549 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T05:39:36,587 DEBUG [RS:0;b543b351fe95:42549 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T05:39:36,587 INFO [RS:0;b543b351fe95:42549 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T05:39:36,587 INFO [RS:0;b543b351fe95:42549 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T05:39:36,588 INFO [RS:1;b543b351fe95:45699 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:36,588 INFO [RS:1;b543b351fe95:45699 {}] regionserver.HRegionServer(1482): Serving as b543b351fe95,45699,1733463574985, RpcServer on b543b351fe95/172.17.0.2:45699, sessionid=0x100ae0a5c730003 2024-12-06T05:39:36,588 DEBUG [RS:1;b543b351fe95:45699 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T05:39:36,588 DEBUG [RS:1;b543b351fe95:45699 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b543b351fe95,45699,1733463574985 2024-12-06T05:39:36,588 DEBUG [RS:1;b543b351fe95:45699 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b543b351fe95,45699,1733463574985' 2024-12-06T05:39:36,588 DEBUG [RS:1;b543b351fe95:45699 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T05:39:36,589 DEBUG [RS:1;b543b351fe95:45699 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T05:39:36,589 DEBUG [RS:1;b543b351fe95:45699 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T05:39:36,589 DEBUG [RS:1;b543b351fe95:45699 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T05:39:36,589 DEBUG [RS:1;b543b351fe95:45699 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b543b351fe95,45699,1733463574985 2024-12-06T05:39:36,589 DEBUG [RS:1;b543b351fe95:45699 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b543b351fe95,45699,1733463574985' 2024-12-06T05:39:36,589 DEBUG [RS:1;b543b351fe95:45699 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T05:39:36,590 DEBUG [RS:1;b543b351fe95:45699 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T05:39:36,591 DEBUG [RS:1;b543b351fe95:45699 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T05:39:36,591 INFO [RS:1;b543b351fe95:45699 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T05:39:36,591 INFO [RS:1;b543b351fe95:45699 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T05:39:36,595 WARN [b543b351fe95:44973 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T05:39:36,693 INFO [RS:1;b543b351fe95:45699 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:39:36,693 INFO [RS:0;b543b351fe95:42549 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:39:36,696 INFO [RS:0;b543b351fe95:42549 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b543b351fe95%2C42549%2C1733463574853, suffix=, logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,42549,1733463574853, archiveDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/oldWALs, maxLogs=32 2024-12-06T05:39:36,696 INFO [RS:1;b543b351fe95:45699 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b543b351fe95%2C45699%2C1733463574985, suffix=, logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,45699,1733463574985, archiveDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/oldWALs, maxLogs=32 2024-12-06T05:39:36,712 DEBUG [RS:1;b543b351fe95:45699 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,45699,1733463574985/b543b351fe95%2C45699%2C1733463574985.1733463576699, exclude list is [], retry=0 2024-12-06T05:39:36,713 DEBUG [RS:0;b543b351fe95:42549 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,42549,1733463574853/b543b351fe95%2C42549%2C1733463574853.1733463576699, exclude list is [], retry=0 2024-12-06T05:39:36,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-80ddb16c-ec66-4e39-a44c-0a80fa143251,DISK] 2024-12-06T05:39:36,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39561,DS-4485eaa6-367b-4084-97e1-b834fb8d8596,DISK] 2024-12-06T05:39:36,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-80ddb16c-ec66-4e39-a44c-0a80fa143251,DISK] 2024-12-06T05:39:36,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39561,DS-4485eaa6-367b-4084-97e1-b834fb8d8596,DISK] 2024-12-06T05:39:36,725 INFO [RS:0;b543b351fe95:42549 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,42549,1733463574853/b543b351fe95%2C42549%2C1733463574853.1733463576699 2024-12-06T05:39:36,725 INFO [RS:1;b543b351fe95:45699 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,45699,1733463574985/b543b351fe95%2C45699%2C1733463574985.1733463576699 2024-12-06T05:39:36,725 DEBUG [RS:0;b543b351fe95:42549 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43559:43559),(127.0.0.1/127.0.0.1:44855:44855)] 2024-12-06T05:39:36,726 DEBUG [RS:1;b543b351fe95:45699 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44855:44855),(127.0.0.1/127.0.0.1:43559:43559)] 2024-12-06T05:39:36,849 DEBUG [b543b351fe95:44973 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-06T05:39:36,856 DEBUG [b543b351fe95:44973 {}] balancer.BalancerClusterState(204): Hosts are {b543b351fe95=0} racks are {/default-rack=0} 2024-12-06T05:39:36,862 DEBUG [b543b351fe95:44973 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-06T05:39:36,862 DEBUG [b543b351fe95:44973 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-06T05:39:36,862 DEBUG [b543b351fe95:44973 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-06T05:39:36,862 DEBUG [b543b351fe95:44973 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-06T05:39:36,862 INFO [b543b351fe95:44973 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-06T05:39:36,862 INFO [b543b351fe95:44973 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-06T05:39:36,862 DEBUG [b543b351fe95:44973 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-06T05:39:36,869 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b543b351fe95,45699,1733463574985 2024-12-06T05:39:36,875 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b543b351fe95,45699,1733463574985, state=OPENING 2024-12-06T05:39:36,879 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-06T05:39:36,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44973-0x100ae0a5c730001, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:36,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45699-0x100ae0a5c730003, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:36,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x100ae0a5c730000, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:36,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42549-0x100ae0a5c730002, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:36,881 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:36,881 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:36,881 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:36,881 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:36,883 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T05:39:36,884 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b543b351fe95,45699,1733463574985}] 2024-12-06T05:39:37,063 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T05:39:37,066 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35047, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T05:39:37,077 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-06T05:39:37,077 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:39:37,077 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-06T05:39:37,081 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b543b351fe95%2C45699%2C1733463574985.meta, suffix=.meta, logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,45699,1733463574985, archiveDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/oldWALs, maxLogs=32 2024-12-06T05:39:37,096 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,45699,1733463574985/b543b351fe95%2C45699%2C1733463574985.meta.1733463577083.meta, exclude list is [], retry=0 2024-12-06T05:39:37,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39561,DS-4485eaa6-367b-4084-97e1-b834fb8d8596,DISK] 2024-12-06T05:39:37,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-80ddb16c-ec66-4e39-a44c-0a80fa143251,DISK] 2024-12-06T05:39:37,102 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,45699,1733463574985/b543b351fe95%2C45699%2C1733463574985.meta.1733463577083.meta 2024-12-06T05:39:37,103 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44855:44855),(127.0.0.1/127.0.0.1:43559:43559)] 2024-12-06T05:39:37,103 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:39:37,104 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-06T05:39:37,107 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-06T05:39:37,112 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-06T05:39:37,117 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-06T05:39:37,117 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:37,118 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-06T05:39:37,118 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-06T05:39:37,121 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-06T05:39:37,122 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T05:39:37,122 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:37,123 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:37,123 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-06T05:39:37,125 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T05:39:37,125 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:37,126 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:37,126 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-06T05:39:37,127 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T05:39:37,127 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:37,128 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:37,128 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-06T05:39:37,130 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T05:39:37,130 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:37,130 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:37,131 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T05:39:37,132 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/hbase/meta/1588230740 2024-12-06T05:39:37,135 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/hbase/meta/1588230740 2024-12-06T05:39:37,138 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T05:39:37,138 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T05:39:37,139 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T05:39:37,142 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T05:39:37,143 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72334606, jitterRate=0.07786962389945984}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T05:39:37,144 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-06T05:39:37,145 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733463577118Writing region info on filesystem at 1733463577118Initializing all the Stores at 1733463577120 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463577120Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463577121 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463577121Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463577121Cleaning up temporary data from old regions at 1733463577138 (+17 ms)Running coprocessor post-open hooks at 1733463577144 (+6 ms)Region opened successfully at 1733463577145 (+1 ms) 2024-12-06T05:39:37,152 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733463577054 2024-12-06T05:39:37,162 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-06T05:39:37,163 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-06T05:39:37,165 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b543b351fe95,45699,1733463574985 2024-12-06T05:39:37,167 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b543b351fe95,45699,1733463574985, state=OPEN 2024-12-06T05:39:37,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x100ae0a5c730000, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:39:37,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44973-0x100ae0a5c730001, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:39:37,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45699-0x100ae0a5c730003, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:39:37,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42549-0x100ae0a5c730002, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:39:37,169 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:37,169 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:37,169 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:37,169 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:37,169 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b543b351fe95,45699,1733463574985 2024-12-06T05:39:37,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-06T05:39:37,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b543b351fe95,45699,1733463574985 in 286 msec 2024-12-06T05:39:37,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-06T05:39:37,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 748 msec 2024-12-06T05:39:37,183 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T05:39:37,184 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-06T05:39:37,200 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T05:39:37,202 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b543b351fe95,45699,1733463574985, seqNum=-1] 2024-12-06T05:39:37,219 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T05:39:37,222 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51687, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T05:39:37,240 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0210 sec 2024-12-06T05:39:37,240 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733463577240, completionTime=-1 2024-12-06T05:39:37,244 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-06T05:39:37,244 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-06T05:39:37,306 INFO [master/b543b351fe95:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-06T05:39:37,306 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733463637306 2024-12-06T05:39:37,306 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733463697306 2024-12-06T05:39:37,306 INFO [master/b543b351fe95:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 62 msec 2024-12-06T05:39:37,308 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-06T05:39:37,316 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,44973,1733463574761-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:37,316 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,44973,1733463574761-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:37,316 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,44973,1733463574761-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:37,322 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b543b351fe95:44973, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:37,322 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:37,323 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:37,330 DEBUG [master/b543b351fe95:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-06T05:39:37,358 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.294sec 2024-12-06T05:39:37,360 INFO [master/b543b351fe95:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-06T05:39:37,362 INFO [master/b543b351fe95:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-06T05:39:37,363 INFO [master/b543b351fe95:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-06T05:39:37,363 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-06T05:39:37,363 INFO [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-06T05:39:37,364 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,44973,1733463574761-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T05:39:37,365 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,44973,1733463574761-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-06T05:39:37,370 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-06T05:39:37,372 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-06T05:39:37,373 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,44973,1733463574761-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:37,424 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x19a8645e to 127.0.0.1:58086 with session timeout=1000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4c87f74d 2024-12-06T05:39:37,425 WARN [Time-limited test {}] client.ZKConnectionRegistry(87): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry 2024-12-06T05:39:37,431 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@144cf54f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T05:39:37,435 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-06T05:39:37,435 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-06T05:39:37,438 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T05:39:37,442 DEBUG [ReadOnlyZKClient-127.0.0.1:58086@0x19a8645e {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b543b351fe95,45699,1733463574985, seqNum=-1] 2024-12-06T05:39:37,442 DEBUG [ReadOnlyZKClient-127.0.0.1:58086@0x19a8645e {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T05:39:37,451 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59468, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T05:39:37,477 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=b543b351fe95,44973,1733463574761 2024-12-06T05:39:37,477 INFO [Time-limited test {}] hbase.TestZooKeeper(120): Starting testRegionServerSessionExpired 2024-12-06T05:39:37,482 INFO [Time-limited test-EventThread {}] hbase.HBaseTestingUtil$1(2484): Monitor ZKW received event=WatchedEvent state:SyncConnected type:None path:null 2024-12-06T05:39:37,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44973-0x100ae0a5c730001, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T05:39:37,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x100ae0a5c730000, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T05:39:37,488 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b543b351fe95,42549,1733463574853] 2024-12-06T05:39:37,489 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44973 {}] master.ServerManager(480): Server REPORT rejected; currently processing b543b351fe95,42549,1733463574853 as dead server 2024-12-06T05:39:37,489 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b543b351fe95,42549,1733463574853 already deleted, retry=false 2024-12-06T05:39:37,490 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of b543b351fe95,42549,1733463574853 on b543b351fe95,44973,1733463574761 2024-12-06T05:39:37,490 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44973 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.YouAreDeadException: Server REPORT rejected; currently processing b543b351fe95,42549,1733463574853 as dead server at org.apache.hadoop.hbase.master.ServerManager.checkIsDead(ServerManager.java:481) ~[classes/:?] at org.apache.hadoop.hbase.master.ServerManager.regionServerReport(ServerManager.java:318) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:643) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:37,494 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44973 {}] ipc.CallRunner(138): callId: 9 service: RegionServerStatusService methodName: RegionServerReport size: 1.3 K connection: 172.17.0.2:46555 deadline: 1733463587489, exception=org.apache.hadoop.hbase.YouAreDeadException: Server REPORT rejected; currently processing b543b351fe95,42549,1733463574853 as dead server 2024-12-06T05:39:37,497 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure b543b351fe95,42549,1733463574853, splitWal=true, meta=false 2024-12-06T05:39:37,497 ERROR [RS:0;b543b351fe95:42549 {}] regionserver.HRegionServer(2440): ***** ABORTING region server b543b351fe95,42549,1733463574853: org.apache.hadoop.hbase.YouAreDeadException: Server REPORT rejected; currently processing b543b351fe95,42549,1733463574853 as dead server at org.apache.hadoop.hbase.master.ServerManager.checkIsDead(ServerManager.java:481) at org.apache.hadoop.hbase.master.ServerManager.regionServerReport(ServerManager.java:318) at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:643) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ***** org.apache.hadoop.hbase.YouAreDeadException: org.apache.hadoop.hbase.YouAreDeadException: Server REPORT rejected; currently processing b543b351fe95,42549,1733463574853 as dead server at org.apache.hadoop.hbase.master.ServerManager.checkIsDead(ServerManager.java:481) at org.apache.hadoop.hbase.master.ServerManager.regionServerReport(ServerManager.java:318) at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:643) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.tryRegionServerReport(HRegionServer.java:1082) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:902) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) ~[test-classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) ~[test-classes/:?] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) ~[test-classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.YouAreDeadException: Server REPORT rejected; currently processing b543b351fe95,42549,1733463574853 as dead server at org.apache.hadoop.hbase.master.ServerManager.checkIsDead(ServerManager.java:481) at org.apache.hadoop.hbase.master.ServerManager.regionServerReport(ServerManager.java:318) at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:643) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-12-06T05:39:37,498 ERROR [RS:0;b543b351fe95:42549 {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-06T05:39:37,503 DEBUG [RS:0;b543b351fe95:42549 {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-06T05:39:37,504 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(169): Start pid=4, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure b543b351fe95,42549,1733463574853, splitWal=true, meta=false 2024-12-06T05:39:37,504 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=4 for b543b351fe95,42549,1733463574853 (carryingMeta=false) b543b351fe95,42549,1733463574853/CRASHED/regionCount=0/lock=java.util.concurrent.locks.ReentrantReadWriteLock@494c034e[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-06T05:39:37,507 DEBUG [RS:0;b543b351fe95:42549 {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-06T05:39:37,507 DEBUG [RS:0;b543b351fe95:42549 {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-06T05:39:37,507 DEBUG [RS:0;b543b351fe95:42549 {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-06T05:39:37,508 INFO [RS:0;b543b351fe95:42549 {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 201920680 }, "NonHeapMemoryUsage": { "committed": 138018816, "init": 7667712, "max": -1, "used": 135730056 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95" } ] } 2024-12-06T05:39:37,511 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): b543b351fe95,42549,1733463574853 had 0 regions 2024-12-06T05:39:37,514 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44973 {}] master.MasterRpcServices(700): b543b351fe95,42549,1733463574853 reported a fatal error: ***** ABORTING region server b543b351fe95,42549,1733463574853: org.apache.hadoop.hbase.YouAreDeadException: Server REPORT rejected; currently processing b543b351fe95,42549,1733463574853 as dead server at org.apache.hadoop.hbase.master.ServerManager.checkIsDead(ServerManager.java:481) at org.apache.hadoop.hbase.master.ServerManager.regionServerReport(ServerManager.java:318) at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:643) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ***** Cause: org.apache.hadoop.hbase.YouAreDeadException: org.apache.hadoop.hbase.YouAreDeadException: Server REPORT rejected; currently processing b543b351fe95,42549,1733463574853 as dead server at org.apache.hadoop.hbase.master.ServerManager.checkIsDead(ServerManager.java:481) at org.apache.hadoop.hbase.master.ServerManager.regionServerReport(ServerManager.java:318) at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:643) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) at org.apache.hadoop.hbase.regionserver.HRegionServer.tryRegionServerReport(HRegionServer.java:1082) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:902) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.YouAreDeadException): org.apache.hadoop.hbase.YouAreDeadException: Server REPORT rejected; currently processing b543b351fe95,42549,1733463574853 as dead server at org.apache.hadoop.hbase.master.ServerManager.checkIsDead(ServerManager.java:481) at org.apache.hadoop.hbase.master.ServerManager.regionServerReport(ServerManager.java:318) at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:643) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-12-06T05:39:37,517 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=4, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure b543b351fe95,42549,1733463574853, splitWal=true, meta=false, isMeta: false 2024-12-06T05:39:37,519 INFO [RS:0;b543b351fe95:42549 {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b543b351fe95,42549,1733463574853' ***** 2024-12-06T05:39:37,519 INFO [RS:0;b543b351fe95:42549 {}] regionserver.HRegionServer(2210): STOPPED: org.apache.hadoop.hbase.YouAreDeadException: Server REPORT rejected; currently processing b543b351fe95,42549,1733463574853 as dead server at org.apache.hadoop.hbase.master.ServerManager.checkIsDead(ServerManager.java:481) at org.apache.hadoop.hbase.master.ServerManager.regionServerReport(ServerManager.java:318) at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:643) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T05:39:37,519 INFO [RS:0;b543b351fe95:42549 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T05:39:37,520 INFO [RS:0;b543b351fe95:42549 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-06T05:39:37,520 INFO [RS:0;b543b351fe95:42549 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-06T05:39:37,520 INFO [RS:0;b543b351fe95:42549 {}] regionserver.HRegionServer(956): aborting server b543b351fe95,42549,1733463574853 2024-12-06T05:39:37,520 INFO [RS:0;b543b351fe95:42549 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T05:39:37,521 INFO [RS:0;b543b351fe95:42549 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;b543b351fe95:42549. 2024-12-06T05:39:37,521 DEBUG [PEWorker-3 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,42549,1733463574853-splitting 2024-12-06T05:39:37,521 DEBUG [RS:0;b543b351fe95:42549 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T05:39:37,521 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T05:39:37,524 INFO [PEWorker-3 {}] master.SplitWALManager(105): b543b351fe95,42549,1733463574853 WAL count=1, meta=false 2024-12-06T05:39:37,524 INFO [regionserver/b543b351fe95:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-06T05:39:37,524 INFO [regionserver/b543b351fe95:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-06T05:39:37,527 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure b543b351fe95%2C42549%2C1733463574853.1733463576699}] 2024-12-06T05:39:37,528 DEBUG [RS:0;b543b351fe95:42549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:39:37,529 INFO [RS:0;b543b351fe95:42549 {}] regionserver.HRegionServer(976): stopping server b543b351fe95,42549,1733463574853; all regions closed. 2024-12-06T05:39:37,537 DEBUG [PEWorker-4 {}] master.SplitWALManager(158): Acquired split WAL worker=b543b351fe95,45699,1733463574985 2024-12-06T05:39:37,540 INFO [regionserver/b543b351fe95:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T05:39:37,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741834_1010 (size=93) 2024-12-06T05:39:37,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741834_1010 (size=93) 2024-12-06T05:39:37,545 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure b543b351fe95%2C42549%2C1733463574853.1733463576699, worker=b543b351fe95,45699,1733463574985}] 2024-12-06T05:39:37,548 DEBUG [RS:0;b543b351fe95:42549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:39:37,548 INFO [RS:0;b543b351fe95:42549 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T05:39:37,548 INFO [RS:0;b543b351fe95:42549 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T05:39:37,549 INFO [RS:0;b543b351fe95:42549 {}] hbase.ChoreService(370): Chore service for: regionserver/b543b351fe95:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-06T05:39:37,549 INFO [RS:0;b543b351fe95:42549 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T05:39:37,549 INFO [RS:0;b543b351fe95:42549 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T05:39:37,549 INFO [RS:0;b543b351fe95:42549 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T05:39:37,549 INFO [RS:0;b543b351fe95:42549 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T05:39:37,550 INFO [RS:0;b543b351fe95:42549 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42549 2024-12-06T05:39:37,552 INFO [regionserver/b543b351fe95:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T05:39:37,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42549-0x100ae0a5c730002, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Disconnected, path=null 2024-12-06T05:39:37,582 DEBUG [RS:0;b543b351fe95:42549 {}] zookeeper.RecoverableZooKeeper(294): Retry, connectivity issue (JVM Pause?); quorum=127.0.0.1:58086,exceptionorg.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss for /hbase/rs/b543b351fe95,42549,1733463574853= 2024-12-06T05:39:37,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42549-0x100ae0a5c730002, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Disconnected, path=null 2024-12-06T05:39:37,582 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(640): regionserver:42549-0x100ae0a5c730002, quorum=127.0.0.1:58086, baseZNode=/hbase Received Disconnected from ZooKeeper, ignoring 2024-12-06T05:39:37,582 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(640): regionserver:42549-0x100ae0a5c730002, quorum=127.0.0.1:58086, baseZNode=/hbase Received Disconnected from ZooKeeper, ignoring 2024-12-06T05:39:37,584 INFO [Time-limited test-EventThread {}] hbase.HBaseTestingUtil$1(2484): Monitor ZKW received event=WatchedEvent state:Disconnected type:None path:null 2024-12-06T05:39:37,588 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2501): ZK Closed Session 0x100ae0a5c730002 2024-12-06T05:39:37,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45699 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=6 2024-12-06T05:39:37,738 INFO [RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0-0 {event_type=RS_LOG_REPLAY, pid=6}] wal.WALSplitter(299): Splitting hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,42549,1733463574853-splitting/b543b351fe95%2C42549%2C1733463574853.1733463576699, size=93 (93bytes) 2024-12-06T05:39:37,739 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0-0 {event_type=RS_LOG_REPLAY, pid=6}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-06T05:39:37,740 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0-0 {event_type=RS_LOG_REPLAY, pid=6}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-06T05:39:37,740 INFO [RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0-0 {event_type=RS_LOG_REPLAY, pid=6}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,42549,1733463574853-splitting/b543b351fe95%2C42549%2C1733463574853.1733463576699 2024-12-06T05:39:37,746 INFO [RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0-0 {event_type=RS_LOG_REPLAY, pid=6}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,42549,1733463574853-splitting/b543b351fe95%2C42549%2C1733463574853.1733463576699 after 4ms 2024-12-06T05:39:37,753 INFO [RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0-0 {event_type=RS_LOG_REPLAY, pid=6}] wal.WALSplitter(310): Open hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,42549,1733463574853-splitting/b543b351fe95%2C42549%2C1733463574853.1733463576699 took 16ms 2024-12-06T05:39:37,755 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0-0 {event_type=RS_LOG_REPLAY, pid=6}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,42549,1733463574853-splitting/b543b351fe95%2C42549%2C1733463574853.1733463576699 so closing down 2024-12-06T05:39:37,755 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0-0 {event_type=RS_LOG_REPLAY, pid=6}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-06T05:39:37,755 INFO [RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0-0 {event_type=RS_LOG_REPLAY, pid=6}] wal.OutputSink(145): 3 split writer threads finished 2024-12-06T05:39:37,760 INFO [RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0-0 {event_type=RS_LOG_REPLAY, pid=6}] wal.WALSplitter(425): Processed 0 edits across 0 Regions in 1 ms; skipped=0; WAL=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,42549,1733463574853-splitting/b543b351fe95%2C42549%2C1733463574853.1733463576699, size=93, length=93, corrupted=false, cancelled=false 2024-12-06T05:39:37,760 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0-0 {event_type=RS_LOG_REPLAY, pid=6}] wal.WALSplitter(428): Completed split of hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,42549,1733463574853-splitting/b543b351fe95%2C42549%2C1733463574853.1733463576699, journal: Splitting hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,42549,1733463574853-splitting/b543b351fe95%2C42549%2C1733463574853.1733463576699, size=93 (93bytes) at 1733463577739Finishing writing output for hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,42549,1733463574853-splitting/b543b351fe95%2C42549%2C1733463574853.1733463576699 so closing down at 1733463577755 (+16 ms)3 split writer threads finished at 1733463577756 (+1 ms)Processed 0 edits across 0 Regions in 1 ms; skipped=0; WAL=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,42549,1733463574853-splitting/b543b351fe95%2C42549%2C1733463574853.1733463576699, size=93, length=93, corrupted=false, cancelled=false at 1733463577760 (+4 ms) 2024-12-06T05:39:37,760 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0-0 {event_type=RS_LOG_REPLAY, pid=6}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,42549,1733463574853-splitting/b543b351fe95%2C42549%2C1733463574853.1733463576699 2024-12-06T05:39:37,762 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0-0 {event_type=RS_LOG_REPLAY, pid=6}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=6 2024-12-06T05:39:37,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44973 {}] master.HMaster(4169): Remote procedure done, pid=6 2024-12-06T05:39:37,787 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,42549,1733463574853-splitting/b543b351fe95%2C42549%2C1733463574853.1733463576699 to hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/oldWALs 2024-12-06T05:39:37,804 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-06T05:39:37,804 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b543b351fe95%2C42549%2C1733463574853.1733463576699, worker=b543b351fe95,45699,1733463574985 in 243 msec 2024-12-06T05:39:37,808 DEBUG [PEWorker-2 {}] master.SplitWALManager(172): Release split WAL worker=b543b351fe95,45699,1733463574985 2024-12-06T05:39:37,816 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-06T05:39:37,817 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; SplitWALProcedure b543b351fe95%2C42549%2C1733463574853.1733463576699, worker=b543b351fe95,45699,1733463574985 in 284 msec 2024-12-06T05:39:37,822 INFO [PEWorker-3 {}] master.SplitLogManager(171): hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,42549,1733463574853-splitting dir is empty, no logs to split. 2024-12-06T05:39:37,822 INFO [PEWorker-3 {}] master.SplitWALManager(105): b543b351fe95,42549,1733463574853 WAL count=0, meta=false 2024-12-06T05:39:37,822 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if b543b351fe95,42549,1733463574853 WAL splitting is done? wals=0, meta=false 2024-12-06T05:39:37,828 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for b543b351fe95,42549,1733463574853 failed, ignore...File hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,42549,1733463574853-splitting does not exist. 2024-12-06T05:39:37,835 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(291): removed crashed server b543b351fe95,42549,1733463574853 after splitting done 2024-12-06T05:39:37,840 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; ServerCrashProcedure b543b351fe95,42549,1733463574853, splitWal=true, meta=false in 344 msec 2024-12-06T05:39:39,407 INFO [Time-limited test-EventThread {}] hbase.HBaseTestingUtil$1(2484): Monitor ZKW received event=WatchedEvent state:Expired type:None path:null 2024-12-06T05:39:39,509 INFO [Time-limited test {}] hbase.TestZooKeeper(153): Creating table testRegionServerSessionExpired_1733463579508 2024-12-06T05:39:39,515 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T05:39:39,517 DEBUG [ReadOnlyZKClient-127.0.0.1:58086@0x19a8645e {}] client.AsyncConnectionImpl(321): The fetched master address is b543b351fe95,44973,1733463574761 2024-12-06T05:39:39,519 DEBUG [ReadOnlyZKClient-127.0.0.1:58086@0x19a8645e {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4dc833e5 2024-12-06T05:39:39,520 DEBUG [ReadOnlyZKClient-127.0.0.1:58086@0x19a8645e {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T05:39:39,523 INFO [HMaster-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49380, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T05:39:39,530 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44973 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRegionServerSessionExpired_1733463579508', {NAME => 'fam', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T05:39:39,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44973 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRegionServerSessionExpired_1733463579508 2024-12-06T05:39:39,539 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRegionServerSessionExpired_1733463579508 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T05:39:39,541 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44973 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRegionServerSessionExpired_1733463579508" procId is: 7 2024-12-06T05:39:39,541 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:39,543 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRegionServerSessionExpired_1733463579508 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T05:39:39,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44973 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-06T05:39:39,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42549-0x100ae0a5c730002, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Expired, path=null 2024-12-06T05:39:39,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42549-0x100ae0a5c730002, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Expired, path=null 2024-12-06T05:39:39,551 DEBUG [zk-event-processor-pool-0 {}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: regionserver:42549-0x100ae0a5c730002, quorum=127.0.0.1:58086, baseZNode=/hbase regionserver:42549-0x100ae0a5c730002 received expired from ZooKeeper, aborting 2024-12-06T05:39:39,551 DEBUG [zk-event-processor-pool-0 {}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: regionserver:42549-0x100ae0a5c730002, quorum=127.0.0.1:58086, baseZNode=/hbase regionserver:42549-0x100ae0a5c730002 received expired from ZooKeeper, aborting 2024-12-06T05:39:39,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741836_1012 (size=395) 2024-12-06T05:39:39,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741836_1012 (size=395) 2024-12-06T05:39:39,558 INFO [RegionOpenAndInit-testRegionServerSessionExpired_1733463579508-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e9944d02e43cb38b96e94f149e46ab64, NAME => 'testRegionServerSessionExpired_1733463579508,,1733463579525.e9944d02e43cb38b96e94f149e46ab64.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionServerSessionExpired_1733463579508', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'fam', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:39:39,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741837_1013 (size=79) 2024-12-06T05:39:39,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741837_1013 (size=79) 2024-12-06T05:39:39,568 DEBUG [RegionOpenAndInit-testRegionServerSessionExpired_1733463579508-pool-0 {}] regionserver.HRegion(898): Instantiated testRegionServerSessionExpired_1733463579508,,1733463579525.e9944d02e43cb38b96e94f149e46ab64.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:39,568 DEBUG [RegionOpenAndInit-testRegionServerSessionExpired_1733463579508-pool-0 {}] regionserver.HRegion(1722): Closing e9944d02e43cb38b96e94f149e46ab64, disabling compactions & flushes 2024-12-06T05:39:39,568 INFO [RegionOpenAndInit-testRegionServerSessionExpired_1733463579508-pool-0 {}] regionserver.HRegion(1755): Closing region testRegionServerSessionExpired_1733463579508,,1733463579525.e9944d02e43cb38b96e94f149e46ab64. 2024-12-06T05:39:39,568 DEBUG [RegionOpenAndInit-testRegionServerSessionExpired_1733463579508-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionServerSessionExpired_1733463579508,,1733463579525.e9944d02e43cb38b96e94f149e46ab64. 2024-12-06T05:39:39,568 DEBUG [RegionOpenAndInit-testRegionServerSessionExpired_1733463579508-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRegionServerSessionExpired_1733463579508,,1733463579525.e9944d02e43cb38b96e94f149e46ab64. after waiting 0 ms 2024-12-06T05:39:39,569 DEBUG [RegionOpenAndInit-testRegionServerSessionExpired_1733463579508-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRegionServerSessionExpired_1733463579508,,1733463579525.e9944d02e43cb38b96e94f149e46ab64. 2024-12-06T05:39:39,569 INFO [RegionOpenAndInit-testRegionServerSessionExpired_1733463579508-pool-0 {}] regionserver.HRegion(1973): Closed testRegionServerSessionExpired_1733463579508,,1733463579525.e9944d02e43cb38b96e94f149e46ab64. 2024-12-06T05:39:39,569 DEBUG [RegionOpenAndInit-testRegionServerSessionExpired_1733463579508-pool-0 {}] regionserver.HRegion(1676): Region close journal for e9944d02e43cb38b96e94f149e46ab64: Waiting for close lock at 1733463579568Disabling compacts and flushes for region at 1733463579568Disabling writes for close at 1733463579569 (+1 ms)Writing region close event to WAL at 1733463579569Closed at 1733463579569 2024-12-06T05:39:39,571 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRegionServerSessionExpired_1733463579508 execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T05:39:39,574 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRegionServerSessionExpired_1733463579508,,1733463579525.e9944d02e43cb38b96e94f149e46ab64.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733463579571"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733463579571"}]},"ts":"1733463579571"} 2024-12-06T05:39:39,578 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T05:39:39,580 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRegionServerSessionExpired_1733463579508 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T05:39:39,583 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRegionServerSessionExpired_1733463579508","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733463579581"}]},"ts":"1733463579581"} 2024-12-06T05:39:39,587 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRegionServerSessionExpired_1733463579508, state=ENABLING in hbase:meta 2024-12-06T05:39:39,589 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionServerSessionExpired_1733463579508, region=e9944d02e43cb38b96e94f149e46ab64, ASSIGN}] 2024-12-06T05:39:39,592 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionServerSessionExpired_1733463579508, region=e9944d02e43cb38b96e94f149e46ab64, ASSIGN 2024-12-06T05:39:39,593 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRegionServerSessionExpired_1733463579508, region=e9944d02e43cb38b96e94f149e46ab64, ASSIGN; state=OFFLINE, location=b543b351fe95,45699,1733463574985; forceNewPlan=false, retain=false 2024-12-06T05:39:39,651 WARN [RS:0;b543b351fe95:42549 {}] regionserver.HRegionServer(1023): Failed deleting my ephemeral node org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /hbase/rs/b543b351fe95,42549,1733463574853 at org.apache.zookeeper.KeeperException.create(KeeperException.java:134) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.delete(ZooKeeper.java:1614) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.delete(RecoverableZooKeeper.java:201) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.deleteNode(ZKUtil.java:851) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.deleteNode(ZKUtil.java:843) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.deleteMyEphemeralNode(HRegionServer.java:1516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:1019) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) ~[test-classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) ~[test-classes/:?] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) ~[test-classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:39:39,651 INFO [RS:0;b543b351fe95:42549 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T05:39:39,651 INFO [RS:0;b543b351fe95:42549 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T05:39:39,651 INFO [RS:0;b543b351fe95:42549 {}] regionserver.HRegionServer(1031): Exiting; stopping=b543b351fe95,42549,1733463574853; zookeeper connection closed. 2024-12-06T05:39:39,652 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3f1be21e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3f1be21e 2024-12-06T05:39:39,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44973 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-06T05:39:39,745 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=e9944d02e43cb38b96e94f149e46ab64, regionState=OPENING, regionLocation=b543b351fe95,45699,1733463574985 2024-12-06T05:39:39,748 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRegionServerSessionExpired_1733463579508, region=e9944d02e43cb38b96e94f149e46ab64, ASSIGN because future has completed 2024-12-06T05:39:39,750 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure e9944d02e43cb38b96e94f149e46ab64, server=b543b351fe95,45699,1733463574985}] 2024-12-06T05:39:39,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44973 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-06T05:39:39,915 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testRegionServerSessionExpired_1733463579508,,1733463579525.e9944d02e43cb38b96e94f149e46ab64. 2024-12-06T05:39:39,915 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => e9944d02e43cb38b96e94f149e46ab64, NAME => 'testRegionServerSessionExpired_1733463579508,,1733463579525.e9944d02e43cb38b96e94f149e46ab64.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:39:39,916 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRegionServerSessionExpired_1733463579508 e9944d02e43cb38b96e94f149e46ab64 2024-12-06T05:39:39,916 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testRegionServerSessionExpired_1733463579508,,1733463579525.e9944d02e43cb38b96e94f149e46ab64.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:39,916 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for e9944d02e43cb38b96e94f149e46ab64 2024-12-06T05:39:39,916 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for e9944d02e43cb38b96e94f149e46ab64 2024-12-06T05:39:39,920 INFO [StoreOpener-e9944d02e43cb38b96e94f149e46ab64-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family fam of region e9944d02e43cb38b96e94f149e46ab64 2024-12-06T05:39:39,922 INFO [StoreOpener-e9944d02e43cb38b96e94f149e46ab64-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9944d02e43cb38b96e94f149e46ab64 columnFamilyName fam 2024-12-06T05:39:39,922 DEBUG [StoreOpener-e9944d02e43cb38b96e94f149e46ab64-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:39,924 INFO [StoreOpener-e9944d02e43cb38b96e94f149e46ab64-1 {}] regionserver.HStore(327): Store=e9944d02e43cb38b96e94f149e46ab64/fam, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:39:39,924 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for e9944d02e43cb38b96e94f149e46ab64 2024-12-06T05:39:39,925 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionServerSessionExpired_1733463579508/e9944d02e43cb38b96e94f149e46ab64 2024-12-06T05:39:39,926 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionServerSessionExpired_1733463579508/e9944d02e43cb38b96e94f149e46ab64 2024-12-06T05:39:39,927 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for e9944d02e43cb38b96e94f149e46ab64 2024-12-06T05:39:39,927 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for e9944d02e43cb38b96e94f149e46ab64 2024-12-06T05:39:39,929 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for e9944d02e43cb38b96e94f149e46ab64 2024-12-06T05:39:39,934 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionServerSessionExpired_1733463579508/e9944d02e43cb38b96e94f149e46ab64/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:39:39,935 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened e9944d02e43cb38b96e94f149e46ab64; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63287555, jitterRate=-0.05694194138050079}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:39:39,935 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e9944d02e43cb38b96e94f149e46ab64 2024-12-06T05:39:39,937 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for e9944d02e43cb38b96e94f149e46ab64: Running coprocessor pre-open hook at 1733463579916Writing region info on filesystem at 1733463579916Initializing all the Stores at 1733463579919 (+3 ms)Instantiating store for column family {NAME => 'fam', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463579920 (+1 ms)Cleaning up temporary data from old regions at 1733463579927 (+7 ms)Running coprocessor post-open hooks at 1733463579935 (+8 ms)Region opened successfully at 1733463579937 (+2 ms) 2024-12-06T05:39:39,939 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testRegionServerSessionExpired_1733463579508,,1733463579525.e9944d02e43cb38b96e94f149e46ab64., pid=9, masterSystemTime=1733463579905 2024-12-06T05:39:39,942 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testRegionServerSessionExpired_1733463579508,,1733463579525.e9944d02e43cb38b96e94f149e46ab64. 2024-12-06T05:39:39,942 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testRegionServerSessionExpired_1733463579508,,1733463579525.e9944d02e43cb38b96e94f149e46ab64. 2024-12-06T05:39:39,943 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=e9944d02e43cb38b96e94f149e46ab64, regionState=OPEN, openSeqNum=2, regionLocation=b543b351fe95,45699,1733463574985 2024-12-06T05:39:39,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure e9944d02e43cb38b96e94f149e46ab64, server=b543b351fe95,45699,1733463574985 because future has completed 2024-12-06T05:39:39,952 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-06T05:39:39,952 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure e9944d02e43cb38b96e94f149e46ab64, server=b543b351fe95,45699,1733463574985 in 198 msec 2024-12-06T05:39:39,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-06T05:39:39,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRegionServerSessionExpired_1733463579508, region=e9944d02e43cb38b96e94f149e46ab64, ASSIGN in 363 msec 2024-12-06T05:39:39,956 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRegionServerSessionExpired_1733463579508 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T05:39:39,957 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRegionServerSessionExpired_1733463579508","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733463579956"}]},"ts":"1733463579956"} 2024-12-06T05:39:39,959 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testRegionServerSessionExpired_1733463579508, state=ENABLED in hbase:meta 2024-12-06T05:39:39,961 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRegionServerSessionExpired_1733463579508 execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T05:39:39,963 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRegionServerSessionExpired_1733463579508 in 428 msec 2024-12-06T05:39:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44973 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-06T05:39:40,181 INFO [Time-limited test {}] hbase.TestZooKeeper(164): Putting table testRegionServerSessionExpired_1733463579508 2024-12-06T05:39:40,181 INFO [RPCClient-NioEventLoopGroup-6-2 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRegionServerSessionExpired_1733463579508 completed 2024-12-06T05:39:40,196 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRegionServerSessionExpired_1733463579508', row='testrow', locateType=CURRENT is [region=testRegionServerSessionExpired_1733463579508,,1733463579525.e9944d02e43cb38b96e94f149e46ab64., hostname=b543b351fe95,45699,1733463574985, seqNum=2] 2024-12-06T05:39:40,215 ERROR [Time-limited test {}] master.HMaster(3225): ***** ABORTING master b543b351fe95,42763,1733463574228: killAll ***** 2024-12-06T05:39:40,215 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master 'b543b351fe95,42763,1733463574228' ***** 2024-12-06T05:39:40,215 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: Stopped by Time-limited test 2024-12-06T05:39:40,215 INFO [M:0;b543b351fe95:42763 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T05:39:40,215 INFO [M:0;b543b351fe95:42763 {}] hbase.ChoreService(370): Chore service for: master/b543b351fe95:0 had [] on shutdown 2024-12-06T05:39:40,215 INFO [M:0;b543b351fe95:42763 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T05:39:40,215 DEBUG [M:0;b543b351fe95:42763 {}] master.HMaster(1795): Stopping service threads 2024-12-06T05:39:40,217 ERROR [Time-limited test {}] master.HMaster(3218): Master server abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-06T05:39:40,217 ERROR [Time-limited test {}] master.HMaster(3225): ***** ABORTING master b543b351fe95,44973,1733463574761: killAll ***** 2024-12-06T05:39:40,217 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master 'b543b351fe95,44973,1733463574761' ***** 2024-12-06T05:39:40,217 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: Stopped by Time-limited test 2024-12-06T05:39:40,217 INFO [M:1;b543b351fe95:44973 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T05:39:40,217 INFO [M:1;b543b351fe95:44973 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:1;b543b351fe95:44973. 2024-12-06T05:39:40,217 DEBUG [M:1;b543b351fe95:44973 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T05:39:40,217 DEBUG [M:1;b543b351fe95:44973 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:39:40,217 INFO [M:1;b543b351fe95:44973 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T05:39:40,218 DEBUG [M:1;b543b351fe95:44973 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-06T05:39:40,218 DEBUG [M:1;b543b351fe95:44973 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-06T05:39:40,218 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-06T05:39:40,218 DEBUG [master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.large.0-1733463576353 {}] cleaner.HFileCleaner(306): Exit Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.large.0-1733463576353,5,FailOnTimeoutGroup] 2024-12-06T05:39:40,218 DEBUG [master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.small.0-1733463576354 {}] cleaner.HFileCleaner(306): Exit Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.small.0-1733463576354,5,FailOnTimeoutGroup] 2024-12-06T05:39:40,218 INFO [M:1;b543b351fe95:44973 {}] hbase.ChoreService(370): Chore service for: master/b543b351fe95:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-06T05:39:40,218 INFO [M:1;b543b351fe95:44973 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T05:39:40,218 DEBUG [M:1;b543b351fe95:44973 {}] master.HMaster(1795): Stopping service threads 2024-12-06T05:39:40,218 INFO [M:1;b543b351fe95:44973 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-06T05:39:40,218 INFO [M:1;b543b351fe95:44973 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-06T05:39:40,219 INFO [M:1;b543b351fe95:44973 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=true 2024-12-06T05:39:40,220 INFO [M:0;b543b351fe95:42763 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42763 2024-12-06T05:39:40,220 INFO [M:0;b543b351fe95:42763 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T05:39:40,220 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-06T05:39:40,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44973-0x100ae0a5c730001, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:40,224 DEBUG [M:1;b543b351fe95:44973 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-12-06T05:39:40,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45699-0x100ae0a5c730003, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:40,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45699-0x100ae0a5c730003, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:40,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45699-0x100ae0a5c730003, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:40,224 DEBUG [M:1;b543b351fe95:44973 {}] master.ActiveMasterManager(353): master:44973-0x100ae0a5c730001, quorum=127.0.0.1:58086, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-12-06T05:39:40,224 DEBUG [Time-limited test {}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: killAll 2024-12-06T05:39:40,224 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server b543b351fe95,45699,1733463574985: killAll ***** 2024-12-06T05:39:40,224 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-06T05:39:40,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x100ae0a5c730000, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b543b351fe95,42763,1733463574228 2024-12-06T05:39:40,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x100ae0a5c730000, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:40,224 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-06T05:39:40,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x100ae0a5c730000, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:40,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x100ae0a5c730000, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:40,225 INFO [M:1;b543b351fe95:44973 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/.lastflushedseqids 2024-12-06T05:39:40,225 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-06T05:39:40,225 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-06T05:39:40,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44973-0x100ae0a5c730001, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:40,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44973-0x100ae0a5c730001, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:40,226 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-06T05:39:40,227 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 254349480 }, "NonHeapMemoryUsage": { "committed": 142540800, "init": 7667712, "max": -1, "used": 140273944 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95" } ] } 2024-12-06T05:39:40,228 DEBUG [HMaster-EventLoopGroup-3-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b543b351fe95,44973,1733463574761 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:39:40,231 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45699-0x100ae0a5c730003, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:39:40,232 WARN [Time-limited test {}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.regionserver.RegionServerAbortedException): Server b543b351fe95,44973,1733463574761 aborting at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:?] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:?] at org.apache.hadoop.hbase.Abortable.abort(Abortable.java:45) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster.killAll(SingleProcessHBaseCluster.java:880) ~[test-classes/:?] at org.apache.hadoop.hbase.TestZooKeeper.after(TestZooKeeper.java:107) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: Server b543b351fe95,44973,1733463574761 aborting at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-12-06T05:39:40,234 DEBUG [HMaster-EventLoopGroup-3-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b543b351fe95,44973,1733463574761 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:39:40,239 DEBUG [RS:1;b543b351fe95:45699 {}] regionserver.HRegionServer(2589): No master found; retry 2024-12-06T05:39:40,252 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b543b351fe95,45699,1733463574985' ***** 2024-12-06T05:39:40,252 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: killAll 2024-12-06T05:39:40,252 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T05:39:40,253 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.TestZooKeeper.after(TestZooKeeper.java:110) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T05:39:40,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:39:40,253 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x19a8645e to 127.0.0.1:58086 2024-12-06T05:39:40,253 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-06T05:39:40,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741838_1014 (size=147) 2024-12-06T05:39:40,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741838_1014 (size=147) 2024-12-06T05:39:40,304 INFO [M:1;b543b351fe95:44973 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-06T05:39:40,305 INFO [M:1;b543b351fe95:44973 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=true 2024-12-06T05:39:40,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741830_1006 (size=39788) 2024-12-06T05:39:40,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741830_1006 (size=39788) 2024-12-06T05:39:40,325 INFO [M:0;b543b351fe95:42763 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T05:39:40,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x100ae0a5c730000, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:39:40,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x100ae0a5c730000, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:39:40,326 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T05:39:40,330 DEBUG [M:1;b543b351fe95:44973 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T05:39:40,330 INFO [M:1;b543b351fe95:44973 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:40,330 DEBUG [M:1;b543b351fe95:44973 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:40,330 DEBUG [M:1;b543b351fe95:44973 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T05:39:40,330 DEBUG [M:1;b543b351fe95:44973 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:40,334 ERROR [M:1;b543b351fe95:44973 {}] regionserver.HRegion(1960): Memstore data size is 34567 in region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:40,334 INFO [M:1;b543b351fe95:44973 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:40,334 DEBUG [M:1;b543b351fe95:44973 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733463580330Disabling compacts and flushes for region at 1733463580330Disabling writes for close at 1733463580330Writing region close event to WAL at 1733463580334 (+4 ms)Closed at 1733463580334 2024-12-06T05:39:40,334 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T05:39:40,334 INFO [M:1;b543b351fe95:44973 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-06T05:39:40,335 INFO [M:1;b543b351fe95:44973 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44973 2024-12-06T05:39:40,338 INFO [M:1;b543b351fe95:44973 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T05:39:40,439 INFO [RS:1;b543b351fe95:45699 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T05:39:40,439 INFO [RS:1;b543b351fe95:45699 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-06T05:39:40,440 INFO [RS:1;b543b351fe95:45699 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-06T05:39:40,440 INFO [RS:1;b543b351fe95:45699 {}] regionserver.HRegionServer(3091): Received CLOSE for e9944d02e43cb38b96e94f149e46ab64 2024-12-06T05:39:40,440 INFO [M:1;b543b351fe95:44973 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T05:39:40,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44973-0x100ae0a5c730001, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:39:40,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44973-0x100ae0a5c730001, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:39:40,444 INFO [RS:1;b543b351fe95:45699 {}] regionserver.HRegionServer(956): aborting server b543b351fe95,45699,1733463574985 2024-12-06T05:39:40,444 INFO [RS:1;b543b351fe95:45699 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T05:39:40,445 INFO [RS:1;b543b351fe95:45699 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;b543b351fe95:45699. 2024-12-06T05:39:40,445 DEBUG [RS:1;b543b351fe95:45699 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T05:39:40,445 DEBUG [RS:1;b543b351fe95:45699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:39:40,445 INFO [RS:1;b543b351fe95:45699 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T05:39:40,445 INFO [RS:1;b543b351fe95:45699 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T05:39:40,445 INFO [RS:1;b543b351fe95:45699 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T05:39:40,445 INFO [RS:1;b543b351fe95:45699 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-06T05:39:40,445 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e9944d02e43cb38b96e94f149e46ab64, disabling compactions & flushes 2024-12-06T05:39:40,446 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRegionServerSessionExpired_1733463579508,,1733463579525.e9944d02e43cb38b96e94f149e46ab64. 2024-12-06T05:39:40,446 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionServerSessionExpired_1733463579508,,1733463579525.e9944d02e43cb38b96e94f149e46ab64. 2024-12-06T05:39:40,446 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRegionServerSessionExpired_1733463579508,,1733463579525.e9944d02e43cb38b96e94f149e46ab64. after waiting 0 ms 2024-12-06T05:39:40,446 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRegionServerSessionExpired_1733463579508,,1733463579525.e9944d02e43cb38b96e94f149e46ab64. 2024-12-06T05:39:40,451 INFO [RS:1;b543b351fe95:45699 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-06T05:39:40,451 DEBUG [RS:1;b543b351fe95:45699 {}] regionserver.HRegionServer(1325): Online Regions={e9944d02e43cb38b96e94f149e46ab64=testRegionServerSessionExpired_1733463579508,,1733463579525.e9944d02e43cb38b96e94f149e46ab64., 1588230740=hbase:meta,,1.1588230740} 2024-12-06T05:39:40,451 DEBUG [RS:1;b543b351fe95:45699 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, e9944d02e43cb38b96e94f149e46ab64 2024-12-06T05:39:40,452 ERROR [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1960): Memstore data size is 41 in region testRegionServerSessionExpired_1733463579508,,1733463579525.e9944d02e43cb38b96e94f149e46ab64. 2024-12-06T05:39:40,465 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T05:39:40,465 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRegionServerSessionExpired_1733463579508,,1733463579525.e9944d02e43cb38b96e94f149e46ab64. 2024-12-06T05:39:40,465 INFO [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T05:39:40,465 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T05:39:40,465 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e9944d02e43cb38b96e94f149e46ab64: Waiting for close lock at 1733463580445Running coprocessor pre-close hooks at 1733463580445Disabling compacts and flushes for region at 1733463580445Disabling writes for close at 1733463580446 (+1 ms)Writing region close event to WAL at 1733463580452 (+6 ms)Running coprocessor post-close hooks at 1733463580452Closed at 1733463580465 (+13 ms) 2024-12-06T05:39:40,465 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T05:39:40,465 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T05:39:40,466 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRegionServerSessionExpired_1733463579508,,1733463579525.e9944d02e43cb38b96e94f149e46ab64. 2024-12-06T05:39:40,469 ERROR [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 1797 in region hbase:meta,,1.1588230740 2024-12-06T05:39:40,469 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-06T05:39:40,469 INFO [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T05:39:40,469 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733463580465Running coprocessor pre-close hooks at 1733463580465Disabling compacts and flushes for region at 1733463580465Disabling writes for close at 1733463580465Writing region close event to WAL at 1733463580469 (+4 ms)Running coprocessor post-close hooks at 1733463580469Closed at 1733463580469 2024-12-06T05:39:40,469 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-06T05:39:40,538 INFO [regionserver/b543b351fe95:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T05:39:40,562 INFO [regionserver/b543b351fe95:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-06T05:39:40,562 INFO [regionserver/b543b351fe95:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-06T05:39:40,652 INFO [RS:1;b543b351fe95:45699 {}] regionserver.HRegionServer(976): stopping server b543b351fe95,45699,1733463574985; all regions closed. 2024-12-06T05:39:40,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741835_1011 (size=2439) 2024-12-06T05:39:40,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741835_1011 (size=2439) 2024-12-06T05:39:40,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741833_1009 (size=607) 2024-12-06T05:39:40,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741833_1009 (size=607) 2024-12-06T05:39:40,680 DEBUG [RS:1;b543b351fe95:45699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:39:40,680 INFO [RS:1;b543b351fe95:45699 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T05:39:40,681 INFO [RS:1;b543b351fe95:45699 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T05:39:40,681 INFO [RS:1;b543b351fe95:45699 {}] hbase.ChoreService(370): Chore service for: regionserver/b543b351fe95:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-06T05:39:40,681 INFO [RS:1;b543b351fe95:45699 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T05:39:40,681 INFO [regionserver/b543b351fe95:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T05:39:40,681 INFO [RS:1;b543b351fe95:45699 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45699 2024-12-06T05:39:40,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45699-0x100ae0a5c730003, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b543b351fe95,45699,1733463574985 2024-12-06T05:39:40,683 INFO [RS:1;b543b351fe95:45699 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T05:39:40,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45699-0x100ae0a5c730003, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:39:40,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45699-0x100ae0a5c730003, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:39:40,785 INFO [RS:1;b543b351fe95:45699 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T05:39:40,785 INFO [RS:1;b543b351fe95:45699 {}] regionserver.HRegionServer(1031): Exiting; stopping=b543b351fe95,45699,1733463574985; zookeeper connection closed. 2024-12-06T05:39:40,801 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@44a56c04 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@44a56c04 2024-12-06T05:39:40,802 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 2 master(s) and 2 regionserver(s) complete 2024-12-06T05:39:40,802 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:40,811 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=testing utility connecting to ZooKeeper ensemble=127.0.0.1:58086 2024-12-06T05:39:40,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility0x0, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:39:40,816 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): testing utility-0x100ae0a5c730005 connected 2024-12-06T05:39:40,820 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-06T05:39:40,821 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/meta-region-server 2024-12-06T05:39:40,822 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/rs 2024-12-06T05:39:40,823 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/splitWAL 2024-12-06T05:39:40,824 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters 2024-12-06T05:39:40,825 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc 2024-12-06T05:39:40,827 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/draining 2024-12-06T05:39:40,830 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/master-maintenance 2024-12-06T05:39:40,833 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot 2024-12-06T05:39:40,834 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/hbaseid 2024-12-06T05:39:40,834 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/table 2024-12-06T05:39:40,846 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc/abort 2024-12-06T05:39:40,848 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc/acquired 2024-12-06T05:39:40,849 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc/reached 2024-12-06T05:39:40,864 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot/abort 2024-12-06T05:39:40,874 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot/acquired 2024-12-06T05:39:40,877 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot/reached 2024-12-06T05:39:40,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/reached 2024-12-06T05:39:40,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/online-snapshot 2024-12-06T05:39:40,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/acquired 2024-12-06T05:39:40,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/abort 2024-12-06T05:39:40,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/reached 2024-12-06T05:39:40,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/flush-table-proc 2024-12-06T05:39:40,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/acquired 2024-12-06T05:39:40,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/abort 2024-12-06T05:39:40,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/table 2024-12-06T05:39:40,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:40,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/hbaseid 2024-12-06T05:39:40,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot 2024-12-06T05:39:40,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master-maintenance 2024-12-06T05:39:40,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/draining 2024-12-06T05:39:40,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc 2024-12-06T05:39:40,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:40,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/splitWAL 2024-12-06T05:39:40,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs 2024-12-06T05:39:40,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:39:40,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T05:39:40,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase 2024-12-06T05:39:40,928 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestZooKeeper#testRegionServerSessionExpired Thread=289 (was 236) Potentially hanging thread: NIOWorkerThread-13 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/b543b351fe95:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: zk-event-processor-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@653dec95 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (268780456) connection to localhost/127.0.0.1:34453 from jenkins.hfs.0 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: NIOWorkerThread-14 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NIOWorkerThread-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NIOWorkerThread-12 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:34453 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NIOWorkerThread-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NIOWorkerThread-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NIOWorkerThread-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NIOWorkerThread-15 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (268780456) connection to localhost/127.0.0.1:34453 from jenkins.hfs.1 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Time-limited test-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Abort regionserver monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: NIOWorkerThread-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NIOWorkerThread-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NIOWorkerThread-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/b543b351fe95:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: NIOWorkerThread-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/b543b351fe95:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-574417408_22 at /127.0.0.1:55766 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NIOWorkerThread-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NIOWorkerThread-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:34453 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test-SendThread(127.0.0.1:58086) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: NIOWorkerThread-16 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:34453 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/b543b351fe95:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Abort regionserver monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=549 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=303 (was 312), ProcessCount=11 (was 11), AvailableMemoryMB=5101 (was 5579) 2024-12-06T05:39:40,942 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestZooKeeper#testMasterSessionExpired Thread=289, OpenFileDescriptor=551, MaxFileDescriptor=1048576, SystemLoadAverage=303, ProcessCount=11, AvailableMemoryMB=5101 2024-12-06T05:39:40,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741839_1015 (size=7) 2024-12-06T05:39:40,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741839_1015 (size=7) 2024-12-06T05:39:40,983 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 with version=8 2024-12-06T05:39:40,983 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/hbase-staging 2024-12-06T05:39:40,987 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b543b351fe95:0 server-side Connection retries=45 2024-12-06T05:39:40,987 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:40,987 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:40,987 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T05:39:40,987 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:40,987 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T05:39:40,988 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-06T05:39:40,988 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T05:39:40,989 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38211 2024-12-06T05:39:40,991 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38211 connecting to ZooKeeper ensemble=127.0.0.1:58086 2024-12-06T05:39:40,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:382110x0, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:39:40,994 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38211-0x100ae0a5c730006 connected 2024-12-06T05:39:41,016 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:41,019 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:41,026 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T05:39:41,026 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279, hbase.cluster.distributed=false 2024-12-06T05:39:41,045 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T05:39:41,053 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38211 2024-12-06T05:39:41,053 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38211 2024-12-06T05:39:41,069 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38211 2024-12-06T05:39:41,070 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38211 2024-12-06T05:39:41,085 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38211 2024-12-06T05:39:41,087 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b543b351fe95:0 server-side Connection retries=45 2024-12-06T05:39:41,087 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:41,087 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:41,087 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T05:39:41,087 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:41,087 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T05:39:41,087 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-06T05:39:41,087 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T05:39:41,089 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38751 2024-12-06T05:39:41,091 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38751 connecting to ZooKeeper ensemble=127.0.0.1:58086 2024-12-06T05:39:41,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:387510x0, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:39:41,116 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:41,117 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38751-0x100ae0a5c730007 connected 2024-12-06T05:39:41,118 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:41,121 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T05:39:41,122 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279, hbase.cluster.distributed=false 2024-12-06T05:39:41,126 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T05:39:41,127 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38751 2024-12-06T05:39:41,128 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38751 2024-12-06T05:39:41,136 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38751 2024-12-06T05:39:41,138 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38751 2024-12-06T05:39:41,139 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38751 2024-12-06T05:39:41,163 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b543b351fe95:0 server-side Connection retries=45 2024-12-06T05:39:41,163 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:41,163 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:41,163 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T05:39:41,163 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:41,163 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T05:39:41,164 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T05:39:41,164 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T05:39:41,165 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37775 2024-12-06T05:39:41,167 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37775 connecting to ZooKeeper ensemble=127.0.0.1:58086 2024-12-06T05:39:41,168 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:41,172 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:41,182 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:377750x0, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T05:39:41,182 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T05:39:41,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:377750x0, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:39:41,207 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T05:39:41,208 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:377750x0, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:39:41,209 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:377750x0, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T05:39:41,220 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37775-0x100ae0a5c730008 connected 2024-12-06T05:39:41,233 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37775 2024-12-06T05:39:41,233 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37775 2024-12-06T05:39:41,236 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37775 2024-12-06T05:39:41,238 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37775 2024-12-06T05:39:41,239 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37775 2024-12-06T05:39:41,258 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b543b351fe95:0 server-side Connection retries=45 2024-12-06T05:39:41,258 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:41,258 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:41,258 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T05:39:41,258 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:41,258 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T05:39:41,258 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T05:39:41,259 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T05:39:41,266 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46769 2024-12-06T05:39:41,268 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46769 connecting to ZooKeeper ensemble=127.0.0.1:58086 2024-12-06T05:39:41,268 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:41,271 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:41,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:467690x0, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:39:41,280 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46769-0x100ae0a5c730009 connected 2024-12-06T05:39:41,285 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46769-0x100ae0a5c730009, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T05:39:41,288 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T05:39:41,298 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T05:39:41,299 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46769-0x100ae0a5c730009, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:39:41,302 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46769-0x100ae0a5c730009, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T05:39:41,306 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46769 2024-12-06T05:39:41,313 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46769 2024-12-06T05:39:41,326 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46769 2024-12-06T05:39:41,339 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46769 2024-12-06T05:39:41,339 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46769 2024-12-06T05:39:41,356 DEBUG [M:0;b543b351fe95:38211 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b543b351fe95:38211 2024-12-06T05:39:41,356 DEBUG [M:1;b543b351fe95:38751 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:1;b543b351fe95:38751 2024-12-06T05:39:41,356 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b543b351fe95,38751,1733463581086 2024-12-06T05:39:41,357 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b543b351fe95,38211,1733463580986 2024-12-06T05:39:41,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37775-0x100ae0a5c730008, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:41,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:41,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:41,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46769-0x100ae0a5c730009, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:41,365 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b543b351fe95,38211,1733463580986 2024-12-06T05:39:41,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:41,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37775-0x100ae0a5c730008, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:41,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37775-0x100ae0a5c730008, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:41,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:41,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46769-0x100ae0a5c730009, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:41,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46769-0x100ae0a5c730009, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:41,370 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b543b351fe95,38751,1733463581086 2024-12-06T05:39:41,371 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T05:39:41,377 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b543b351fe95,38211,1733463580986 from backup master directory 2024-12-06T05:39:41,379 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T05:39:41,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b543b351fe95,38211,1733463580986 2024-12-06T05:39:41,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37775-0x100ae0a5c730008, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:41,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:41,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:41,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46769-0x100ae0a5c730009, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:41,380 WARN [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T05:39:41,380 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b543b351fe95,38211,1733463580986 2024-12-06T05:39:41,380 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ActiveMasterManager(296): Another master is the active master, b543b351fe95,38211,1733463580986; waiting to become the next active master 2024-12-06T05:39:41,402 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/hbase.id] with ID: f24feb01-2567-4d5e-a4c6-86f800bb290a 2024-12-06T05:39:41,402 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/.tmp/hbase.id 2024-12-06T05:39:41,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741840_1016 (size=42) 2024-12-06T05:39:41,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741840_1016 (size=42) 2024-12-06T05:39:41,461 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/.tmp/hbase.id]:[hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/hbase.id] 2024-12-06T05:39:41,493 INFO [master/b543b351fe95:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:41,493 INFO [master/b543b351fe95:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-06T05:39:41,499 INFO [master/b543b351fe95:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 5ms. 2024-12-06T05:39:41,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:41,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46769-0x100ae0a5c730009, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:41,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:41,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37775-0x100ae0a5c730008, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:41,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741841_1017 (size=196) 2024-12-06T05:39:41,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741841_1017 (size=196) 2024-12-06T05:39:41,568 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T05:39:41,569 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-06T05:39:41,573 INFO [master/b543b351fe95:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:39:41,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741842_1018 (size=1189) 2024-12-06T05:39:41,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741842_1018 (size=1189) 2024-12-06T05:39:41,649 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store 2024-12-06T05:39:41,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741843_1019 (size=34) 2024-12-06T05:39:41,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741843_1019 (size=34) 2024-12-06T05:39:41,710 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:41,710 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T05:39:41,710 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:41,710 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:41,710 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T05:39:41,710 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:41,711 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:41,711 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733463581710Disabling compacts and flushes for region at 1733463581710Disabling writes for close at 1733463581710Writing region close event to WAL at 1733463581710Closed at 1733463581710 2024-12-06T05:39:41,713 WARN [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/.initializing 2024-12-06T05:39:41,713 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,38211,1733463580986 2024-12-06T05:39:41,716 INFO [master/b543b351fe95:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:39:41,719 INFO [master/b543b351fe95:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b543b351fe95%2C38211%2C1733463580986, suffix=, logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,38211,1733463580986, archiveDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/oldWALs, maxLogs=10 2024-12-06T05:39:41,740 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,38211,1733463580986/b543b351fe95%2C38211%2C1733463580986.1733463581720, exclude list is [], retry=0 2024-12-06T05:39:41,747 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39561,DS-4485eaa6-367b-4084-97e1-b834fb8d8596,DISK] 2024-12-06T05:39:41,758 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-80ddb16c-ec66-4e39-a44c-0a80fa143251,DISK] 2024-12-06T05:39:41,778 INFO [master/b543b351fe95:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,38211,1733463580986/b543b351fe95%2C38211%2C1733463580986.1733463581720 2024-12-06T05:39:41,781 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43559:43559),(127.0.0.1/127.0.0.1:44855:44855)] 2024-12-06T05:39:41,781 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:39:41,782 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:41,782 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:41,782 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:41,785 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:41,791 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-06T05:39:41,792 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:41,794 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:41,794 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:41,797 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-06T05:39:41,797 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:41,798 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:39:41,798 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:41,805 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-06T05:39:41,805 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:41,811 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:39:41,811 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:41,815 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-06T05:39:41,815 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:41,816 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:39:41,817 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:41,818 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:41,818 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:41,821 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:41,822 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:41,825 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T05:39:41,835 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:41,850 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:39:41,851 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68988307, jitterRate=0.02800588309764862}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T05:39:41,854 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733463581782Initializing all the Stores at 1733463581784 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463581784Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463581784Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463581784Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463581784Cleaning up temporary data from old regions at 1733463581822 (+38 ms)Region opened successfully at 1733463581853 (+31 ms) 2024-12-06T05:39:41,856 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-06T05:39:41,866 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@618f0117, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b543b351fe95/172.17.0.2:0 2024-12-06T05:39:41,867 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-06T05:39:41,867 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-06T05:39:41,868 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-06T05:39:41,868 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-06T05:39:41,869 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-06T05:39:41,870 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-06T05:39:41,870 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-06T05:39:41,876 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-06T05:39:41,877 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-06T05:39:41,879 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-06T05:39:41,879 INFO [master/b543b351fe95:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-06T05:39:41,881 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-06T05:39:41,883 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-06T05:39:41,884 INFO [master/b543b351fe95:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-06T05:39:41,892 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-06T05:39:41,893 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-06T05:39:41,900 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-06T05:39:41,901 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-06T05:39:41,905 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-06T05:39:41,906 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-06T05:39:41,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T05:39:41,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:41,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46769-0x100ae0a5c730009, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T05:39:41,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46769-0x100ae0a5c730009, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:41,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37775-0x100ae0a5c730008, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T05:39:41,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37775-0x100ae0a5c730008, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:41,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T05:39:41,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:41,910 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=b543b351fe95,38211,1733463580986, sessionid=0x100ae0a5c730006, setting cluster-up flag (Was=false) 2024-12-06T05:39:41,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46769-0x100ae0a5c730009, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:41,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:41,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37775-0x100ae0a5c730008, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:41,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:41,924 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-06T05:39:41,925 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b543b351fe95,38211,1733463580986 2024-12-06T05:39:41,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37775-0x100ae0a5c730008, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:41,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46769-0x100ae0a5c730009, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:41,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:41,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:41,938 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-06T05:39:41,940 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b543b351fe95,38211,1733463580986 2024-12-06T05:39:41,942 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-06T05:39:41,947 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-06T05:39:41,947 INFO [master/b543b351fe95:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-06T05:39:41,948 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b543b351fe95,38211,1733463580986 Number of backup masters: 1 b543b351fe95,38751,1733463581086 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-06T05:39:41,950 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:39:41,950 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:39:41,950 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:39:41,950 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:39:41,950 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b543b351fe95:0, corePoolSize=10, maxPoolSize=10 2024-12-06T05:39:41,950 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:41,950 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b543b351fe95:0, corePoolSize=2, maxPoolSize=2 2024-12-06T05:39:41,951 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:41,975 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T05:39:41,975 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-06T05:39:41,982 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:41,991 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-06T05:39:42,002 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733463612002 2024-12-06T05:39:42,003 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-06T05:39:42,003 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-06T05:39:42,003 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-06T05:39:42,003 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-06T05:39:42,003 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-06T05:39:42,003 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-06T05:39:42,003 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:42,004 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-06T05:39:42,004 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-06T05:39:42,005 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-06T05:39:42,005 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-06T05:39:42,005 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-06T05:39:42,006 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.large.0-1733463582005,5,FailOnTimeoutGroup] 2024-12-06T05:39:42,010 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.small.0-1733463582006,5,FailOnTimeoutGroup] 2024-12-06T05:39:42,010 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:42,010 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-06T05:39:42,010 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:42,010 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:42,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741845_1021 (size=1321) 2024-12-06T05:39:42,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741845_1021 (size=1321) 2024-12-06T05:39:42,034 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-06T05:39:42,034 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:39:42,048 INFO [RS:0;b543b351fe95:37775 {}] regionserver.HRegionServer(746): ClusterId : f24feb01-2567-4d5e-a4c6-86f800bb290a 2024-12-06T05:39:42,048 DEBUG [RS:0;b543b351fe95:37775 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T05:39:42,051 DEBUG [RS:0;b543b351fe95:37775 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T05:39:42,051 DEBUG [RS:0;b543b351fe95:37775 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T05:39:42,056 INFO [RS:1;b543b351fe95:46769 {}] regionserver.HRegionServer(746): ClusterId : f24feb01-2567-4d5e-a4c6-86f800bb290a 2024-12-06T05:39:42,056 DEBUG [RS:1;b543b351fe95:46769 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T05:39:42,057 DEBUG [RS:0;b543b351fe95:37775 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T05:39:42,058 DEBUG [RS:0;b543b351fe95:37775 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59a43b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b543b351fe95/172.17.0.2:0 2024-12-06T05:39:42,059 DEBUG [RS:1;b543b351fe95:46769 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T05:39:42,059 DEBUG [RS:1;b543b351fe95:46769 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T05:39:42,062 DEBUG [RS:1;b543b351fe95:46769 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T05:39:42,062 DEBUG [RS:1;b543b351fe95:46769 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70e3cdcd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b543b351fe95/172.17.0.2:0 2024-12-06T05:39:42,085 DEBUG [RS:0;b543b351fe95:37775 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b543b351fe95:37775 2024-12-06T05:39:42,085 INFO [RS:0;b543b351fe95:37775 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T05:39:42,085 INFO [RS:0;b543b351fe95:37775 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T05:39:42,085 DEBUG [RS:0;b543b351fe95:37775 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T05:39:42,089 INFO [RS:0;b543b351fe95:37775 {}] regionserver.HRegionServer(2659): reportForDuty to master=b543b351fe95,38211,1733463580986 with port=37775, startcode=1733463581162 2024-12-06T05:39:42,091 DEBUG [RS:0;b543b351fe95:37775 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T05:39:42,100 DEBUG [RS:1;b543b351fe95:46769 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;b543b351fe95:46769 2024-12-06T05:39:42,101 INFO [RS:1;b543b351fe95:46769 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T05:39:42,101 INFO [RS:1;b543b351fe95:46769 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T05:39:42,101 DEBUG [RS:1;b543b351fe95:46769 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T05:39:42,102 INFO [RS:1;b543b351fe95:46769 {}] regionserver.HRegionServer(2659): reportForDuty to master=b543b351fe95,38211,1733463580986 with port=46769, startcode=1733463581257 2024-12-06T05:39:42,103 DEBUG [RS:1;b543b351fe95:46769 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T05:39:42,113 INFO [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40955, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T05:39:42,114 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38211 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b543b351fe95,37775,1733463581162 2024-12-06T05:39:42,115 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38211 {}] master.ServerManager(517): Registering regionserver=b543b351fe95,37775,1733463581162 2024-12-06T05:39:42,122 DEBUG [RS:0;b543b351fe95:37775 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:39:42,122 DEBUG [RS:0;b543b351fe95:37775 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34453 2024-12-06T05:39:42,122 DEBUG [RS:0;b543b351fe95:37775 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T05:39:42,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T05:39:42,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T05:39:42,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741846_1022 (size=32) 2024-12-06T05:39:42,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741846_1022 (size=32) 2024-12-06T05:39:42,130 DEBUG [RS:0;b543b351fe95:37775 {}] zookeeper.ZKUtil(111): regionserver:37775-0x100ae0a5c730008, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b543b351fe95,37775,1733463581162 2024-12-06T05:39:42,130 WARN [RS:0;b543b351fe95:37775 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T05:39:42,130 INFO [RS:0;b543b351fe95:37775 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:39:42,130 DEBUG [RS:0;b543b351fe95:37775 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,37775,1733463581162 2024-12-06T05:39:42,131 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:42,137 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-06T05:39:42,143 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T05:39:42,143 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:42,144 INFO [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43501, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T05:39:42,144 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:42,145 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38211 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b543b351fe95,46769,1733463581257 2024-12-06T05:39:42,145 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38211 {}] master.ServerManager(517): Registering regionserver=b543b351fe95,46769,1733463581257 2024-12-06T05:39:42,145 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-06T05:39:42,150 DEBUG [RS:1;b543b351fe95:46769 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:39:42,150 DEBUG [RS:1;b543b351fe95:46769 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34453 2024-12-06T05:39:42,150 DEBUG [RS:1;b543b351fe95:46769 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T05:39:42,153 DEBUG [RS:1;b543b351fe95:46769 {}] zookeeper.ZKUtil(111): regionserver:46769-0x100ae0a5c730009, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b543b351fe95,46769,1733463581257 2024-12-06T05:39:42,153 WARN [RS:1;b543b351fe95:46769 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T05:39:42,153 INFO [RS:1;b543b351fe95:46769 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:39:42,153 DEBUG [RS:1;b543b351fe95:46769 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,46769,1733463581257 2024-12-06T05:39:42,154 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T05:39:42,155 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:42,156 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:42,157 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-06T05:39:42,165 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T05:39:42,165 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:42,166 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:42,167 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-06T05:39:42,170 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T05:39:42,170 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:42,171 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:42,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T05:39:42,173 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/hbase/meta/1588230740 2024-12-06T05:39:42,173 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/hbase/meta/1588230740 2024-12-06T05:39:42,176 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T05:39:42,176 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T05:39:42,177 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T05:39:42,178 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b543b351fe95,37775,1733463581162] 2024-12-06T05:39:42,179 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b543b351fe95,46769,1733463581257] 2024-12-06T05:39:42,181 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T05:39:42,201 INFO [RS:1;b543b351fe95:46769 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T05:39:42,208 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:39:42,210 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71577354, jitterRate=0.06658568978309631}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T05:39:42,211 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733463582131Initializing all the Stores at 1733463582135 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463582135Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463582137 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463582137Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463582137Cleaning up temporary data from old regions at 1733463582176 (+39 ms)Region opened successfully at 1733463582211 (+35 ms) 2024-12-06T05:39:42,211 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T05:39:42,211 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T05:39:42,211 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T05:39:42,211 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T05:39:42,211 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T05:39:42,211 INFO [RS:0;b543b351fe95:37775 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T05:39:42,215 INFO [RS:1;b543b351fe95:46769 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T05:39:42,218 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T05:39:42,218 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733463582211Disabling compacts and flushes for region at 1733463582211Disabling writes for close at 1733463582211Writing region close event to WAL at 1733463582217 (+6 ms)Closed at 1733463582217 2024-12-06T05:39:42,219 INFO [RS:0;b543b351fe95:37775 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T05:39:42,220 INFO [RS:0;b543b351fe95:37775 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T05:39:42,220 INFO [RS:0;b543b351fe95:37775 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:42,221 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T05:39:42,221 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-06T05:39:42,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-06T05:39:42,223 INFO [RS:0;b543b351fe95:37775 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T05:39:42,225 INFO [RS:0;b543b351fe95:37775 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T05:39:42,225 INFO [RS:0;b543b351fe95:37775 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:42,225 DEBUG [RS:0;b543b351fe95:37775 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:42,226 DEBUG [RS:0;b543b351fe95:37775 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:42,226 DEBUG [RS:0;b543b351fe95:37775 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:42,226 INFO [RS:1;b543b351fe95:46769 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T05:39:42,226 INFO [RS:1;b543b351fe95:46769 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:42,226 DEBUG [RS:0;b543b351fe95:37775 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:42,226 DEBUG [RS:0;b543b351fe95:37775 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:42,226 DEBUG [RS:0;b543b351fe95:37775 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0, corePoolSize=2, maxPoolSize=2 2024-12-06T05:39:42,226 DEBUG [RS:0;b543b351fe95:37775 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:42,227 DEBUG [RS:0;b543b351fe95:37775 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:42,227 DEBUG [RS:0;b543b351fe95:37775 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:42,227 DEBUG [RS:0;b543b351fe95:37775 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:42,227 DEBUG [RS:0;b543b351fe95:37775 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:42,227 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T05:39:42,227 DEBUG [RS:0;b543b351fe95:37775 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:42,227 DEBUG [RS:0;b543b351fe95:37775 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b543b351fe95:0, corePoolSize=3, maxPoolSize=3 2024-12-06T05:39:42,227 DEBUG [RS:0;b543b351fe95:37775 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b543b351fe95:0, corePoolSize=3, maxPoolSize=3 2024-12-06T05:39:42,229 INFO [RS:1;b543b351fe95:46769 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T05:39:42,231 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-06T05:39:42,233 INFO [RS:1;b543b351fe95:46769 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T05:39:42,233 INFO [RS:1;b543b351fe95:46769 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:42,233 DEBUG [RS:1;b543b351fe95:46769 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:42,233 DEBUG [RS:1;b543b351fe95:46769 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:42,233 DEBUG [RS:1;b543b351fe95:46769 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:42,234 DEBUG [RS:1;b543b351fe95:46769 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:42,234 DEBUG [RS:1;b543b351fe95:46769 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:42,234 DEBUG [RS:1;b543b351fe95:46769 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0, corePoolSize=2, maxPoolSize=2 2024-12-06T05:39:42,234 DEBUG [RS:1;b543b351fe95:46769 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:42,234 DEBUG [RS:1;b543b351fe95:46769 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:42,234 DEBUG [RS:1;b543b351fe95:46769 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:42,234 DEBUG [RS:1;b543b351fe95:46769 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:42,234 DEBUG [RS:1;b543b351fe95:46769 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:42,234 DEBUG [RS:1;b543b351fe95:46769 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:42,234 DEBUG [RS:1;b543b351fe95:46769 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b543b351fe95:0, corePoolSize=3, maxPoolSize=3 2024-12-06T05:39:42,234 DEBUG [RS:1;b543b351fe95:46769 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b543b351fe95:0, corePoolSize=3, maxPoolSize=3 2024-12-06T05:39:42,257 INFO [RS:0;b543b351fe95:37775 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:42,257 INFO [RS:0;b543b351fe95:37775 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:42,257 INFO [RS:0;b543b351fe95:37775 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:42,257 INFO [RS:0;b543b351fe95:37775 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:42,257 INFO [RS:0;b543b351fe95:37775 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:42,257 INFO [RS:0;b543b351fe95:37775 {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,37775,1733463581162-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T05:39:42,269 INFO [RS:1;b543b351fe95:46769 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:42,269 INFO [RS:1;b543b351fe95:46769 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:42,269 INFO [RS:1;b543b351fe95:46769 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:42,270 INFO [RS:1;b543b351fe95:46769 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:42,270 INFO [RS:1;b543b351fe95:46769 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:42,270 INFO [RS:1;b543b351fe95:46769 {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,46769,1733463581257-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T05:39:42,290 INFO [RS:1;b543b351fe95:46769 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T05:39:42,290 INFO [RS:1;b543b351fe95:46769 {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,46769,1733463581257-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:42,290 INFO [RS:1;b543b351fe95:46769 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:42,291 INFO [RS:1;b543b351fe95:46769 {}] regionserver.Replication(171): b543b351fe95,46769,1733463581257 started 2024-12-06T05:39:42,291 INFO [RS:0;b543b351fe95:37775 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T05:39:42,291 INFO [RS:0;b543b351fe95:37775 {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,37775,1733463581162-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:42,291 INFO [RS:0;b543b351fe95:37775 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:42,292 INFO [RS:0;b543b351fe95:37775 {}] regionserver.Replication(171): b543b351fe95,37775,1733463581162 started 2024-12-06T05:39:42,316 INFO [RS:1;b543b351fe95:46769 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:42,317 INFO [RS:1;b543b351fe95:46769 {}] regionserver.HRegionServer(1482): Serving as b543b351fe95,46769,1733463581257, RpcServer on b543b351fe95/172.17.0.2:46769, sessionid=0x100ae0a5c730009 2024-12-06T05:39:42,317 DEBUG [RS:1;b543b351fe95:46769 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T05:39:42,317 DEBUG [RS:1;b543b351fe95:46769 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b543b351fe95,46769,1733463581257 2024-12-06T05:39:42,317 DEBUG [RS:1;b543b351fe95:46769 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b543b351fe95,46769,1733463581257' 2024-12-06T05:39:42,317 DEBUG [RS:1;b543b351fe95:46769 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T05:39:42,318 DEBUG [RS:1;b543b351fe95:46769 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T05:39:42,319 DEBUG [RS:1;b543b351fe95:46769 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T05:39:42,319 DEBUG [RS:1;b543b351fe95:46769 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T05:39:42,319 DEBUG [RS:1;b543b351fe95:46769 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b543b351fe95,46769,1733463581257 2024-12-06T05:39:42,319 DEBUG [RS:1;b543b351fe95:46769 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b543b351fe95,46769,1733463581257' 2024-12-06T05:39:42,319 DEBUG [RS:1;b543b351fe95:46769 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T05:39:42,320 DEBUG [RS:1;b543b351fe95:46769 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T05:39:42,320 DEBUG [RS:1;b543b351fe95:46769 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T05:39:42,321 INFO [RS:1;b543b351fe95:46769 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T05:39:42,321 INFO [RS:1;b543b351fe95:46769 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T05:39:42,324 INFO [RS:0;b543b351fe95:37775 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:42,324 INFO [RS:0;b543b351fe95:37775 {}] regionserver.HRegionServer(1482): Serving as b543b351fe95,37775,1733463581162, RpcServer on b543b351fe95/172.17.0.2:37775, sessionid=0x100ae0a5c730008 2024-12-06T05:39:42,324 DEBUG [RS:0;b543b351fe95:37775 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T05:39:42,324 DEBUG [RS:0;b543b351fe95:37775 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b543b351fe95,37775,1733463581162 2024-12-06T05:39:42,325 DEBUG [RS:0;b543b351fe95:37775 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b543b351fe95,37775,1733463581162' 2024-12-06T05:39:42,325 DEBUG [RS:0;b543b351fe95:37775 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T05:39:42,327 DEBUG [RS:0;b543b351fe95:37775 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T05:39:42,327 DEBUG [RS:0;b543b351fe95:37775 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T05:39:42,328 DEBUG [RS:0;b543b351fe95:37775 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T05:39:42,328 DEBUG [RS:0;b543b351fe95:37775 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b543b351fe95,37775,1733463581162 2024-12-06T05:39:42,328 DEBUG [RS:0;b543b351fe95:37775 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b543b351fe95,37775,1733463581162' 2024-12-06T05:39:42,328 DEBUG [RS:0;b543b351fe95:37775 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T05:39:42,329 DEBUG [RS:0;b543b351fe95:37775 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T05:39:42,330 DEBUG [RS:0;b543b351fe95:37775 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T05:39:42,330 INFO [RS:0;b543b351fe95:37775 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T05:39:42,330 INFO [RS:0;b543b351fe95:37775 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T05:39:42,382 WARN [b543b351fe95:38211 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T05:39:42,422 INFO [RS:1;b543b351fe95:46769 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:39:42,427 INFO [RS:1;b543b351fe95:46769 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b543b351fe95%2C46769%2C1733463581257, suffix=, logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,46769,1733463581257, archiveDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/oldWALs, maxLogs=32 2024-12-06T05:39:42,431 INFO [RS:0;b543b351fe95:37775 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:39:42,438 INFO [RS:0;b543b351fe95:37775 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b543b351fe95%2C37775%2C1733463581162, suffix=, logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,37775,1733463581162, archiveDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/oldWALs, maxLogs=32 2024-12-06T05:39:42,463 DEBUG [RS:0;b543b351fe95:37775 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,37775,1733463581162/b543b351fe95%2C37775%2C1733463581162.1733463582440, exclude list is [], retry=0 2024-12-06T05:39:42,463 DEBUG [RS:1;b543b351fe95:46769 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,46769,1733463581257/b543b351fe95%2C46769%2C1733463581257.1733463582429, exclude list is [], retry=0 2024-12-06T05:39:42,469 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-80ddb16c-ec66-4e39-a44c-0a80fa143251,DISK] 2024-12-06T05:39:42,469 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39561,DS-4485eaa6-367b-4084-97e1-b834fb8d8596,DISK] 2024-12-06T05:39:42,489 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39561,DS-4485eaa6-367b-4084-97e1-b834fb8d8596,DISK] 2024-12-06T05:39:42,489 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-80ddb16c-ec66-4e39-a44c-0a80fa143251,DISK] 2024-12-06T05:39:42,520 INFO [RS:1;b543b351fe95:46769 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,46769,1733463581257/b543b351fe95%2C46769%2C1733463581257.1733463582429 2024-12-06T05:39:42,523 DEBUG [RS:1;b543b351fe95:46769 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44855:44855),(127.0.0.1/127.0.0.1:43559:43559)] 2024-12-06T05:39:42,524 INFO [RS:0;b543b351fe95:37775 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,37775,1733463581162/b543b351fe95%2C37775%2C1733463581162.1733463582440 2024-12-06T05:39:42,526 DEBUG [RS:0;b543b351fe95:37775 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44855:44855),(127.0.0.1/127.0.0.1:43559:43559)] 2024-12-06T05:39:42,632 DEBUG [b543b351fe95:38211 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-06T05:39:42,633 DEBUG [b543b351fe95:38211 {}] balancer.BalancerClusterState(204): Hosts are {b543b351fe95=0} racks are {/default-rack=0} 2024-12-06T05:39:42,635 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T05:39:42,644 DEBUG [b543b351fe95:38211 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-06T05:39:42,644 DEBUG [b543b351fe95:38211 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-06T05:39:42,644 DEBUG [b543b351fe95:38211 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-06T05:39:42,644 DEBUG [b543b351fe95:38211 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-06T05:39:42,644 INFO [b543b351fe95:38211 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-06T05:39:42,644 INFO [b543b351fe95:38211 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-06T05:39:42,644 DEBUG [b543b351fe95:38211 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-06T05:39:42,645 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b543b351fe95,37775,1733463581162 2024-12-06T05:39:42,647 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b543b351fe95,37775,1733463581162, state=OPENING 2024-12-06T05:39:42,684 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-06T05:39:42,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46769-0x100ae0a5c730009, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:42,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37775-0x100ae0a5c730008, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:42,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:42,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:42,729 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:42,729 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:42,729 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:42,730 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T05:39:42,730 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:42,730 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b543b351fe95,37775,1733463581162}] 2024-12-06T05:39:42,889 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T05:39:42,894 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38891, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T05:39:42,912 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-06T05:39:42,912 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:39:42,913 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-06T05:39:42,921 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b543b351fe95%2C37775%2C1733463581162.meta, suffix=.meta, logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,37775,1733463581162, archiveDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/oldWALs, maxLogs=32 2024-12-06T05:39:42,950 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,37775,1733463581162/b543b351fe95%2C37775%2C1733463581162.meta.1733463582922.meta, exclude list is [], retry=0 2024-12-06T05:39:42,959 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39561,DS-4485eaa6-367b-4084-97e1-b834fb8d8596,DISK] 2024-12-06T05:39:42,959 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-80ddb16c-ec66-4e39-a44c-0a80fa143251,DISK] 2024-12-06T05:39:42,985 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,37775,1733463581162/b543b351fe95%2C37775%2C1733463581162.meta.1733463582922.meta 2024-12-06T05:39:42,986 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43559:43559),(127.0.0.1/127.0.0.1:44855:44855)] 2024-12-06T05:39:42,987 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:39:42,987 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-06T05:39:42,987 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-06T05:39:42,987 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-06T05:39:42,988 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-06T05:39:42,988 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:42,988 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-06T05:39:42,988 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-06T05:39:42,991 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-06T05:39:42,992 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T05:39:42,993 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:42,994 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:42,994 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-06T05:39:42,996 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T05:39:42,996 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:43,001 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:43,002 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-06T05:39:43,003 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T05:39:43,004 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:43,005 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:43,005 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-06T05:39:43,006 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T05:39:43,006 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:43,007 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:43,008 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T05:39:43,011 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/hbase/meta/1588230740 2024-12-06T05:39:43,016 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/hbase/meta/1588230740 2024-12-06T05:39:43,023 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T05:39:43,023 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T05:39:43,026 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T05:39:43,030 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T05:39:43,033 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59231555, jitterRate=-0.11738105118274689}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T05:39:43,033 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-06T05:39:43,035 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733463582988Writing region info on filesystem at 1733463582988Initializing all the Stores at 1733463582989 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463582990 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463582990Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463582990Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463582990Cleaning up temporary data from old regions at 1733463583023 (+33 ms)Running coprocessor post-open hooks at 1733463583033 (+10 ms)Region opened successfully at 1733463583034 (+1 ms) 2024-12-06T05:39:43,037 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733463582889 2024-12-06T05:39:43,045 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b543b351fe95,37775,1733463581162 2024-12-06T05:39:43,047 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-06T05:39:43,047 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-06T05:39:43,047 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b543b351fe95,37775,1733463581162, state=OPEN 2024-12-06T05:39:43,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:39:43,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46769-0x100ae0a5c730009, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:39:43,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:39:43,048 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:43,048 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:43,048 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:43,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37775-0x100ae0a5c730008, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:39:43,049 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b543b351fe95,37775,1733463581162 2024-12-06T05:39:43,049 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:43,053 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-06T05:39:43,053 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b543b351fe95,37775,1733463581162 in 319 msec 2024-12-06T05:39:43,058 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-06T05:39:43,058 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 832 msec 2024-12-06T05:39:43,059 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T05:39:43,059 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-06T05:39:43,062 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T05:39:43,062 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b543b351fe95,37775,1733463581162, seqNum=-1] 2024-12-06T05:39:43,062 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T05:39:43,064 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45229, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T05:39:43,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1280 sec 2024-12-06T05:39:43,077 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733463583076, completionTime=-1 2024-12-06T05:39:43,077 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-06T05:39:43,077 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-06T05:39:43,085 INFO [master/b543b351fe95:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-06T05:39:43,085 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733463643085 2024-12-06T05:39:43,086 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733463703085 2024-12-06T05:39:43,086 INFO [master/b543b351fe95:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-06T05:39:43,086 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,38211,1733463580986-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:43,086 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,38211,1733463580986-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:43,086 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,38211,1733463580986-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:43,087 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b543b351fe95:38211, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:43,087 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:43,087 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:43,090 DEBUG [master/b543b351fe95:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-06T05:39:43,097 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.717sec 2024-12-06T05:39:43,100 INFO [master/b543b351fe95:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-06T05:39:43,100 INFO [master/b543b351fe95:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-06T05:39:43,100 INFO [master/b543b351fe95:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-06T05:39:43,100 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-06T05:39:43,100 INFO [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-06T05:39:43,100 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,38211,1733463580986-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T05:39:43,100 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,38211,1733463580986-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-06T05:39:43,117 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-06T05:39:43,117 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-06T05:39:43,117 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,38211,1733463580986-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:43,158 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2602908d to 127.0.0.1:58086 with session timeout=1000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7482ff69 2024-12-06T05:39:43,188 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60e14816, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T05:39:43,188 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T05:39:43,191 DEBUG [ReadOnlyZKClient-127.0.0.1:58086@0x2602908d {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b543b351fe95,37775,1733463581162, seqNum=-1] 2024-12-06T05:39:43,192 DEBUG [ReadOnlyZKClient-127.0.0.1:58086@0x2602908d {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T05:39:43,198 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50354, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T05:39:43,203 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=b543b351fe95,38211,1733463580986 2024-12-06T05:39:43,203 INFO [Time-limited test {}] hbase.TestZooKeeper(127): Starting testMasterSessionExpired 2024-12-06T05:39:43,226 INFO [Time-limited test-EventThread {}] hbase.HBaseTestingUtil$1(2484): Monitor ZKW received event=WatchedEvent state:SyncConnected type:None path:null 2024-12-06T05:39:43,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46769-0x100ae0a5c730009, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:43,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37775-0x100ae0a5c730008, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:43,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:43,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46769-0x100ae0a5c730009, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:43,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:43,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37775-0x100ae0a5c730008, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:43,231 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37775-0x100ae0a5c730008, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:39:43,232 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46769-0x100ae0a5c730009, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:39:43,234 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:39:43,234 DEBUG [zk-event-processor-pool-0 {}] master.ActiveMasterManager(204): No master available. Notifying waiting threads 2024-12-06T05:39:43,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46769-0x100ae0a5c730009, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:43,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:43,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37775-0x100ae0a5c730008, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:43,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:43,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37775-0x100ae0a5c730008, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:43,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46769-0x100ae0a5c730009, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:43,242 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T05:39:43,242 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T05:39:43,243 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b543b351fe95,38751,1733463581086 from backup master directory 2024-12-06T05:39:43,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46769-0x100ae0a5c730009, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:43,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37775-0x100ae0a5c730008, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:43,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b543b351fe95,38751,1733463581086 2024-12-06T05:39:43,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:43,244 WARN [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T05:39:43,244 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b543b351fe95,38751,1733463581086 2024-12-06T05:39:43,276 INFO [master/b543b351fe95:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:43,276 INFO [master/b543b351fe95:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-06T05:39:43,295 INFO [master/b543b351fe95:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=1) cost 19ms. 2024-12-06T05:39:43,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Disconnected, path=null 2024-12-06T05:39:43,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Disconnected, path=null 2024-12-06T05:39:43,321 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(640): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Received Disconnected from ZooKeeper, ignoring 2024-12-06T05:39:43,321 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(640): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Received Disconnected from ZooKeeper, ignoring 2024-12-06T05:39:43,329 INFO [Time-limited test-EventThread {}] hbase.HBaseTestingUtil$1(2484): Monitor ZKW received event=WatchedEvent state:Disconnected type:None path:null 2024-12-06T05:39:43,331 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2501): ZK Closed Session 0x100ae0a5c730006 2024-12-06T05:39:43,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741850_1026 (size=196) 2024-12-06T05:39:43,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741850_1026 (size=196) 2024-12-06T05:39:43,361 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T05:39:43,362 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-06T05:39:43,363 INFO [master/b543b351fe95:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:39:43,380 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(342): old store file tracker DEFAULT is the same with new store file tracker, skip migration 2024-12-06T05:39:43,393 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(316): Renamed hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,38211,1733463580986 to hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,38211,1733463580986-dead as it is dead 2024-12-06T05:39:43,394 INFO [master/b543b351fe95:0:becomeActiveMaster {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,38211,1733463580986-dead/b543b351fe95%2C38211%2C1733463580986.1733463581720 2024-12-06T05:39:43,396 WARN [IPC Server handler 1 on default port 34453 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,38211,1733463580986-dead/b543b351fe95%2C38211%2C1733463580986.1733463581720 has not been closed. Lease recovery is in progress. RecoveryId = 1027 for block blk_1073741844_1020 2024-12-06T05:39:43,399 INFO [master/b543b351fe95:0:becomeActiveMaster {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,38211,1733463580986-dead/b543b351fe95%2C38211%2C1733463580986.1733463581720 after 4ms 2024-12-06T05:39:44,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Expired, path=null 2024-12-06T05:39:44,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Expired, path=null 2024-12-06T05:39:44,578 ERROR [zk-event-processor-pool-0 {}] master.HMaster(3218): Master server abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-06T05:39:44,578 ERROR [zk-event-processor-pool-0 {}] master.HMaster(3223): ***** ABORTING master b543b351fe95,38211,1733463580986: master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase master:38211-0x100ae0a5c730006 received expired from ZooKeeper, aborting ***** org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired at org.apache.hadoop.hbase.zookeeper.ZKWatcher.connectionEvent(ZKWatcher.java:652) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.lambda$processEvent$0(ZKWatcher.java:563) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.processEvent(ZKWatcher.java:559) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.lambda$process$1(ZKWatcher.java:613) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:39:44,578 INFO [zk-event-processor-pool-0 {}] master.HMaster(3321): ***** STOPPING master 'b543b351fe95,38211,1733463580986' ***** 2024-12-06T05:39:44,579 INFO [zk-event-processor-pool-0 {}] master.HMaster(3323): STOPPED: Stopped by zk-event-processor-pool-0 2024-12-06T05:39:44,579 WARN [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(351): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/master org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /hbase/master at org.apache.zookeeper.KeeperException.create(KeeperException.java:134) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1972) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:368) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:356) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.getData(ZKUtil.java:343) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.MasterAddressTracker.getMasterAddress(MasterAddressTracker.java:171) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.ActiveMasterManager.stop(ActiveMasterManager.java:342) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.stop(HMaster.java:3327) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.stopMaster(HMaster.java:3315) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.abort(HMaster.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.connectionEvent(ZKWatcher.java:652) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.lambda$processEvent$0(ZKWatcher.java:563) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.processEvent(ZKWatcher.java:559) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.lambda$process$1(ZKWatcher.java:613) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:39:44,579 INFO [M:0;b543b351fe95:38211 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T05:39:44,579 ERROR [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(711): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Received unexpected KeeperException, re-throwing exception org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /hbase/master at org.apache.zookeeper.KeeperException.create(KeeperException.java:134) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1972) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:368) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:356) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.getData(ZKUtil.java:343) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.MasterAddressTracker.getMasterAddress(MasterAddressTracker.java:171) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.ActiveMasterManager.stop(ActiveMasterManager.java:342) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.stop(HMaster.java:3327) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.stopMaster(HMaster.java:3315) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.abort(HMaster.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.connectionEvent(ZKWatcher.java:652) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.lambda$processEvent$0(ZKWatcher.java:563) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.processEvent(ZKWatcher.java:559) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.lambda$process$1(ZKWatcher.java:613) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:39:44,579 INFO [M:0;b543b351fe95:38211 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;b543b351fe95:38211. 2024-12-06T05:39:44,579 DEBUG [zk-event-processor-pool-0 {}] master.ActiveMasterManager(353): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = Session expired for /hbase/master 2024-12-06T05:39:44,579 DEBUG [M:0;b543b351fe95:38211 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T05:39:44,579 DEBUG [M:0;b543b351fe95:38211 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:39:44,579 DEBUG [zk-event-processor-pool-0 {}] master.HMaster(3213): Abort called but aborted=true, stopped=true 2024-12-06T05:39:44,580 INFO [M:0;b543b351fe95:38211 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T05:39:44,580 DEBUG [M:0;b543b351fe95:38211 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-06T05:39:44,580 DEBUG [M:0;b543b351fe95:38211 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-06T05:39:44,580 DEBUG [master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.large.0-1733463582005 {}] cleaner.HFileCleaner(306): Exit Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.large.0-1733463582005,5,FailOnTimeoutGroup] 2024-12-06T05:39:44,580 DEBUG [master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.small.0-1733463582006 {}] cleaner.HFileCleaner(306): Exit Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.small.0-1733463582006,5,FailOnTimeoutGroup] 2024-12-06T05:39:44,580 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-06T05:39:44,581 INFO [M:0;b543b351fe95:38211 {}] hbase.ChoreService(370): Chore service for: master/b543b351fe95:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-06T05:39:44,581 INFO [M:0;b543b351fe95:38211 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T05:39:44,581 DEBUG [M:0;b543b351fe95:38211 {}] master.HMaster(1795): Stopping service threads 2024-12-06T05:39:44,581 INFO [M:0;b543b351fe95:38211 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-06T05:39:44,582 INFO [M:0;b543b351fe95:38211 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-06T05:39:44,583 INFO [M:0;b543b351fe95:38211 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=true 2024-12-06T05:39:44,583 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-06T05:39:44,583 WARN [M:0;b543b351fe95:38211 {}] zookeeper.ZKUtil(351): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/master org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /hbase/master at org.apache.zookeeper.KeeperException.create(KeeperException.java:134) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1972) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:368) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:356) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.getData(ZKUtil.java:343) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.MasterAddressTracker.getMasterAddress(MasterAddressTracker.java:171) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.ActiveMasterManager.stop(ActiveMasterManager.java:342) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.stopServiceThreads(HMaster.java:1809) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:631) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:39:44,583 ERROR [M:0;b543b351fe95:38211 {}] zookeeper.ZKWatcher(711): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Received unexpected KeeperException, re-throwing exception org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /hbase/master at org.apache.zookeeper.KeeperException.create(KeeperException.java:134) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1972) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:368) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:356) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.getData(ZKUtil.java:343) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.MasterAddressTracker.getMasterAddress(MasterAddressTracker.java:171) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.ActiveMasterManager.stop(ActiveMasterManager.java:342) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.stopServiceThreads(HMaster.java:1809) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:631) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:39:44,583 DEBUG [M:0;b543b351fe95:38211 {}] master.ActiveMasterManager(353): master:38211-0x100ae0a5c730006, quorum=127.0.0.1:58086, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = Session expired for /hbase/master 2024-12-06T05:39:44,585 INFO [M:0;b543b351fe95:38211 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/.lastflushedseqids 2024-12-06T05:39:44,592 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-06T05:39:44,593 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-06T05:39:44,595 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-06T05:39:44,595 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-06T05:39:44,595 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-06T05:39:44,595 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-06T05:39:44,600 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b543b351fe95,38211,1733463580986 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:39:44,602 DEBUG [HMaster-EventLoopGroup-7-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b543b351fe95,38211,1733463580986 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:39:44,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741851_1028 (size=99) 2024-12-06T05:39:44,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741851_1028 (size=99) 2024-12-06T05:39:44,630 INFO [M:0;b543b351fe95:38211 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-06T05:39:44,630 INFO [M:0;b543b351fe95:38211 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=true 2024-12-06T05:39:44,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741844_1020 (size=9664) 2024-12-06T05:39:44,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741844_1020 (size=9664) 2024-12-06T05:39:44,643 WARN [Close-WAL-Writer-0 {}] wal.AsyncProtobufLogWriter(165): normal close failed, try recover java.io.IOException: Commit or complete block blk_1073741844_1020, whereas it is under recovery. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.commitOrCompleteLastBlock(BlockManager.java:1211) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.commitOrCompleteLastBlock(FSNamesystem.java:3890) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:731) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:163) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: Commit or complete block blk_1073741844_1020, whereas it is under recovery. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.commitOrCompleteLastBlock(BlockManager.java:1211) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.commitOrCompleteLastBlock(FSNamesystem.java:3890) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:731) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 6 more 2024-12-06T05:39:44,646 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,38211,1733463580986/b543b351fe95%2C38211%2C1733463580986.1733463581720 2024-12-06T05:39:44,647 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,38211,1733463580986/b543b351fe95%2C38211%2C1733463580986.1733463581720 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.recoverLease(FSNamesystem.java:2869) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.recoverLease(NameNodeRpcServer.java:872) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.recoverLease(ClientNamenodeProtocolServerSideTranslatorPB.java:834) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:951) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.recoverAndClose(FanOutOneBlockAsyncDFSOutput.java:605) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:166) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,38211,1733463580986/b543b351fe95%2C38211%2C1733463580986.1733463581720 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.recoverLease(FSNamesystem.java:2869) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.recoverLease(NameNodeRpcServer.java:872) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.recoverLease(ClientNamenodeProtocolServerSideTranslatorPB.java:834) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.recoverLease(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$recoverLease$24(ClientNamenodeProtocolTranslatorPB.java:685) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.recoverLease(ClientNamenodeProtocolTranslatorPB.java:685) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:949) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 17 more 2024-12-06T05:39:44,651 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,38211,1733463580986/b543b351fe95%2C38211%2C1733463580986.1733463581720 2024-12-06T05:39:44,652 ERROR [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2033): Unable to recover lease after several attempts. Give up. java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,38211,1733463580986/b543b351fe95%2C38211%2C1733463580986.1733463581720 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.recoverLease(FSNamesystem.java:2869) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.recoverLease(NameNodeRpcServer.java:872) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.recoverLease(ClientNamenodeProtocolServerSideTranslatorPB.java:834) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:951) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,38211,1733463580986/b543b351fe95%2C38211%2C1733463580986.1733463581720 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.recoverLease(FSNamesystem.java:2869) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.recoverLease(NameNodeRpcServer.java:872) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.recoverLease(ClientNamenodeProtocolServerSideTranslatorPB.java:834) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.recoverLease(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$recoverLease$24(ClientNamenodeProtocolTranslatorPB.java:685) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.recoverLease(ClientNamenodeProtocolTranslatorPB.java:685) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:949) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 16 more 2024-12-06T05:39:44,661 DEBUG [M:0;b543b351fe95:38211 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T05:39:44,661 INFO [M:0;b543b351fe95:38211 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:44,661 DEBUG [M:0;b543b351fe95:38211 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:44,661 DEBUG [M:0;b543b351fe95:38211 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T05:39:44,661 DEBUG [M:0;b543b351fe95:38211 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:44,667 ERROR [M:0;b543b351fe95:38211 {}] regionserver.HRegion(1960): Memstore data size is 7867 in region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:44,667 INFO [M:0;b543b351fe95:38211 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:44,668 DEBUG [M:0;b543b351fe95:38211 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733463584660Disabling compacts and flushes for region at 1733463584660Disabling writes for close at 1733463584661 (+1 ms)Writing region close event to WAL at 1733463584667 (+6 ms)Closed at 1733463584667 2024-12-06T05:39:44,668 INFO [M:0;b543b351fe95:38211 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-06T05:39:44,668 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T05:39:44,668 INFO [M:0;b543b351fe95:38211 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38211 2024-12-06T05:39:44,670 INFO [M:0;b543b351fe95:38211 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T05:39:44,670 INFO [M:0;b543b351fe95:38211 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T05:39:44,705 DEBUG [RS:0;b543b351fe95:37775 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T05:39:44,705 DEBUG [RS:1;b543b351fe95:46769 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T05:39:44,708 INFO [HMaster-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36877, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T05:39:44,710 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:44,715 INFO [HMaster-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53187, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T05:39:44,718 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:44,816 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:44,825 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:44,921 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:44,930 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:45,024 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:45,033 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:45,128 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:45,143 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:45,232 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:45,254 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:45,282 INFO [Time-limited test-EventThread {}] hbase.HBaseTestingUtil$1(2484): Monitor ZKW received event=WatchedEvent state:Expired type:None path:null 2024-12-06T05:39:45,336 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:45,363 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:45,382 INFO [Time-limited test {}] hbase.TestZooKeeper(153): Creating table testMasterSessionExpired_1733463585382 2024-12-06T05:39:45,383 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T05:39:45,384 DEBUG [ReadOnlyZKClient-127.0.0.1:58086@0x2602908d {}] client.AsyncConnectionImpl(321): The fetched master address is b543b351fe95,38751,1733463581086 2024-12-06T05:39:45,384 DEBUG [ReadOnlyZKClient-127.0.0.1:58086@0x2602908d {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4c568ff9 2024-12-06T05:39:45,384 DEBUG [ReadOnlyZKClient-127.0.0.1:58086@0x2602908d {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T05:39:45,387 INFO [HMaster-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53572, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T05:39:45,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.checkInitialized(HMaster.java:3340) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.createTable(HMaster.java:2437) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.createTable(MasterRpcServices.java:786) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:45,440 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:45,466 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:45,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.checkInitialized(HMaster.java:3340) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.createTable(HMaster.java:2437) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.createTable(MasterRpcServices.java:786) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:45,545 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:45,572 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:45,650 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:45,675 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.checkInitialized(HMaster.java:3340) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.createTable(HMaster.java:2437) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.createTable(MasterRpcServices.java:786) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:45,753 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:45,778 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:45,856 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:45,881 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:45,960 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:45,984 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:46,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.checkInitialized(HMaster.java:3340) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.createTable(HMaster.java:2437) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.createTable(MasterRpcServices.java:786) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:46,064 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:46,088 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:46,168 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:46,192 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:46,272 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:46,297 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:46,375 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:46,400 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:46,479 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:46,503 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:46,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.checkInitialized(HMaster.java:3340) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.createTable(HMaster.java:2437) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.createTable(MasterRpcServices.java:786) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:46,582 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:46,606 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:46,689 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:46,710 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:46,792 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:46,813 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:46,896 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:46,917 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:47,000 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:47,020 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:47,103 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:47,122 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:47,206 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:47,228 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:47,309 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:47,331 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:47,400 INFO [master/b543b351fe95:0:becomeActiveMaster {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,38211,1733463580986-dead/b543b351fe95%2C38211%2C1733463580986.1733463581720 after 4006ms 2024-12-06T05:39:47,405 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(328): Renamed hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,38211,1733463580986-dead/b543b351fe95%2C38211%2C1733463580986.1733463581720 to hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.wals/b543b351fe95%2C38211%2C1733463580986.1733463581720 2024-12-06T05:39:47,405 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(330): Delete empty local region wal dir hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,38211,1733463580986-dead 2024-12-06T05:39:47,406 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,38751,1733463581086 2024-12-06T05:39:47,408 INFO [master/b543b351fe95:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:39:47,413 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:47,414 INFO [master/b543b351fe95:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b543b351fe95%2C38751%2C1733463581086, suffix=, logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,38751,1733463581086, archiveDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/oldWALs, maxLogs=10 2024-12-06T05:39:47,434 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:47,435 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,38751,1733463581086/b543b351fe95%2C38751%2C1733463581086.1733463587415, exclude list is [], retry=0 2024-12-06T05:39:47,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39561,DS-4485eaa6-367b-4084-97e1-b834fb8d8596,DISK] 2024-12-06T05:39:47,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-80ddb16c-ec66-4e39-a44c-0a80fa143251,DISK] 2024-12-06T05:39:47,442 INFO [master/b543b351fe95:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,38751,1733463581086/b543b351fe95%2C38751%2C1733463581086.1733463587415 2024-12-06T05:39:47,444 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43559:43559),(127.0.0.1/127.0.0.1:44855:44855)] 2024-12-06T05:39:47,444 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:39:47,444 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:47,445 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:47,445 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:47,450 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:47,451 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-06T05:39:47,452 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:47,452 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:47,453 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:47,454 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-06T05:39:47,454 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:47,455 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:39:47,455 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:47,456 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-06T05:39:47,456 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:47,457 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:39:47,457 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:47,458 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-06T05:39:47,458 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:47,459 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:39:47,459 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:47,460 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(5516): Found 1 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.wals 2024-12-06T05:39:47,460 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.wals/b543b351fe95%2C38211%2C1733463580986.1733463581720 2024-12-06T05:39:47,473 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(5793): Applied 36, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=28, path=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.wals/b543b351fe95%2C38211%2C1733463580986.1733463581720 2024-12-06T05:39:47,477 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.68 KB heapSize=11.36 KB 2024-12-06T05:39:47,516 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:47,537 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.checkInitialized(HMaster.java:3340) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.createTable(HMaster.java:2437) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.createTable(MasterRpcServices.java:786) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:47,544 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0ae96b9bbdb74b9197bd1f64e16b6c54 is 82, key is hbase:meta,,1/info:regioninfo/1733463583045/Put/seqid=0 2024-12-06T05:39:47,545 WARN [RPCClient-NioEventLoopGroup-6-4 {}] client.AsyncRpcRetryingCaller(177): Call to master failed, tries = 6, maxAttempts = 15, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.HMaster.checkInitialized(HMaster.java:3340) at org.apache.hadoop.hbase.master.HMaster.createTable(HMaster.java:2437) at org.apache.hadoop.hbase.master.MasterRpcServices.createTable(MasterRpcServices.java:786) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor57.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:465) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.HMaster.checkInitialized(HMaster.java:3340) at org.apache.hadoop.hbase.master.HMaster.createTable(HMaster.java:2437) at org.apache.hadoop.hbase.master.MasterRpcServices.createTable(MasterRpcServices.java:786) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T05:39:47,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741853_1030 (size=5672) 2024-12-06T05:39:47,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741853_1030 (size=5672) 2024-12-06T05:39:47,620 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:47,640 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:47,724 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:47,743 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:47,828 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:47,848 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:47,932 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:47,951 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:47,973 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0ae96b9bbdb74b9197bd1f64e16b6c54 2024-12-06T05:39:48,034 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:48,044 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e666a6763e7a4091b6e645e99a6b4eda is 366, key is \x00\x00\x00\x00\x00\x00\x00\x02/proc:d/1733463583053/Put/seqid=0 2024-12-06T05:39:48,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741854_1031 (size=12229) 2024-12-06T05:39:48,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741854_1031 (size=12229) 2024-12-06T05:39:48,053 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e666a6763e7a4091b6e645e99a6b4eda 2024-12-06T05:39:48,054 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:48,079 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/03ee375200694f01886df815b5ed5689 is 69, key is b543b351fe95,37775,1733463581162/rs:state/1733463582115/Put/seqid=0 2024-12-06T05:39:48,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741855_1032 (size=5224) 2024-12-06T05:39:48,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741855_1032 (size=5224) 2024-12-06T05:39:48,091 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/03ee375200694f01886df815b5ed5689 2024-12-06T05:39:48,101 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0ae96b9bbdb74b9197bd1f64e16b6c54 as hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0ae96b9bbdb74b9197bd1f64e16b6c54 2024-12-06T05:39:48,110 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0ae96b9bbdb74b9197bd1f64e16b6c54, entries=8, sequenceid=28, filesize=5.5 K 2024-12-06T05:39:48,113 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e666a6763e7a4091b6e645e99a6b4eda as hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e666a6763e7a4091b6e645e99a6b4eda 2024-12-06T05:39:48,125 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e666a6763e7a4091b6e645e99a6b4eda, entries=26, sequenceid=28, filesize=11.9 K 2024-12-06T05:39:48,127 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/03ee375200694f01886df815b5ed5689 as hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/03ee375200694f01886df815b5ed5689 2024-12-06T05:39:48,137 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/03ee375200694f01886df815b5ed5689, entries=2, sequenceid=28, filesize=5.1 K 2024-12-06T05:39:48,137 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:48,143 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.68 KB/7867, heapSize ~11.06 KB/11328, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 665ms, sequenceid=28, compaction requested=false; wal=null 2024-12-06T05:39:48,146 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.wals/b543b351fe95%2C38211%2C1733463580986.1733463581720 2024-12-06T05:39:48,148 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:48,148 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:48,149 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T05:39:48,151 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:48,154 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/28.seqid, newMaxSeqId=28, maxSeqId=1 2024-12-06T05:39:48,156 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=29; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67221640, jitterRate=0.0016804933547973633}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T05:39:48,157 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:48,157 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733463587445Initializing all the Stores at 1733463587447 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463587447Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463587450 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463587450Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463587450Obtaining lock to block concurrent updates at 1733463587477 (+27 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733463587477Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7867, getHeapSize=11568, getOffHeapSize=0, getCellsCount=36 at 1733463587480 (+3 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733463587480Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733463587482 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733463587531 (+49 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733463587534 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733463588020 (+486 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733463588043 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733463588043Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733463588063 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733463588078 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733463588079 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@792d56c2: reopening flushed file at 1733463588099 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d5655f8: reopening flushed file at 1733463588111 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f761427: reopening flushed file at 1733463588125 (+14 ms)Finished flush of dataSize ~7.68 KB/7867, heapSize ~11.06 KB/11328, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 665ms, sequenceid=28, compaction requested=false; wal=null at 1733463588143 (+18 ms)Cleaning up temporary data from old regions at 1733463588148 (+5 ms)Region opened successfully at 1733463588157 (+9 ms) 2024-12-06T05:39:48,158 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-06T05:39:48,163 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594bf595, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b543b351fe95/172.17.0.2:0 2024-12-06T05:39:48,174 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(884): The info family in master local region already has data in it, skip migrating... 2024-12-06T05:39:48,174 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-06T05:39:48,174 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-06T05:39:48,174 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-06T05:39:48,175 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-06T05:39:48,179 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-06T05:39:48,180 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-06T05:39:48,180 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-06T05:39:48,184 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b543b351fe95,37775,1733463581162, regionLocation=b543b351fe95,37775,1733463581162, openSeqNum=2 2024-12-06T05:39:48,184 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b543b351fe95,37775,1733463581162, table=hbase:meta, region=1588230740 2024-12-06T05:39:48,184 INFO [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b543b351fe95,37775,1733463581162, state=OPEN 2024-12-06T05:39:48,240 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:48,260 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:48,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46769-0x100ae0a5c730009, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:39:48,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:39:48,275 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:48,275 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:48,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37775-0x100ae0a5c730008, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:39:48,276 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:48,292 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 2 possibly 'live' servers, and 0 'splitting'. 2024-12-06T05:39:48,293 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ServerManager(517): Registering regionserver=b543b351fe95,37775,1733463581162 2024-12-06T05:39:48,293 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ServerManager(517): Registering regionserver=b543b351fe95,46769,1733463581257 2024-12-06T05:39:48,294 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-06T05:39:48,378 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:48,379 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:48,413 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-06T05:39:48,414 INFO [master/b543b351fe95:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-06T05:39:48,415 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-06T05:39:48,418 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-06T05:39:48,419 INFO [master/b543b351fe95:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-06T05:39:48,422 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-06T05:39:48,425 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-06T05:39:48,427 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-06T05:39:48,427 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-06T05:39:48,433 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-06T05:39:48,434 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-06T05:39:48,434 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=b543b351fe95,38751,1733463581086, sessionid=0x100ae0a5c730007, setting cluster-up flag (Was=true) 2024-12-06T05:39:48,437 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-06T05:39:48,438 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b543b351fe95,38751,1733463581086 2024-12-06T05:39:48,443 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-06T05:39:48,444 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b543b351fe95,38751,1733463581086 2024-12-06T05:39:48,446 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ServerManager(1190): begin to load .lastflushedseqids at hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/.lastflushedseqids 2024-12-06T05:39:48,450 INFO [master/b543b351fe95:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-06T05:39:48,451 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b543b351fe95,38751,1733463581086 Number of backup masters: 0 Number of live region servers: 2 b543b351fe95,37775,1733463581162 b543b351fe95,46769,1733463581257 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-06T05:39:48,453 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:39:48,453 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:39:48,453 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:39:48,454 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:39:48,454 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b543b351fe95:0, corePoolSize=10, maxPoolSize=10 2024-12-06T05:39:48,454 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:48,454 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b543b351fe95:0, corePoolSize=2, maxPoolSize=2 2024-12-06T05:39:48,454 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:48,457 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733463618456 2024-12-06T05:39:48,457 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-06T05:39:48,457 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-06T05:39:48,457 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-06T05:39:48,457 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-06T05:39:48,457 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-06T05:39:48,457 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-06T05:39:48,461 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:48,462 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-06T05:39:48,463 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-06T05:39:48,463 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-06T05:39:48,464 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-06T05:39:48,464 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-06T05:39:48,469 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.large.0-1733463588464,5,FailOnTimeoutGroup] 2024-12-06T05:39:48,485 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.small.0-1733463588469,5,FailOnTimeoutGroup] 2024-12-06T05:39:48,485 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:48,485 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-06T05:39:48,485 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:48,486 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:48,486 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733463588486, completionTime=-1 2024-12-06T05:39:48,486 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-06T05:39:48,486 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-06T05:39:48,486 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T05:39:48,487 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b543b351fe95,37775,1733463581162, seqNum=-1] 2024-12-06T05:39:48,487 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T05:39:48,490 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60169, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T05:39:48,492 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T05:39:48,492 INFO [master/b543b351fe95:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-06T05:39:48,492 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733463648492 2024-12-06T05:39:48,492 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733463708492 2024-12-06T05:39:48,492 INFO [master/b543b351fe95:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-06T05:39:48,493 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,38751,1733463581086-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:48,493 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,38751,1733463581086-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:48,493 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,38751,1733463581086-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:48,493 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b543b351fe95:38751, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:48,493 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:48,497 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:48,502 DEBUG [master/b543b351fe95:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-06T05:39:48,509 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 5.264sec 2024-12-06T05:39:48,512 INFO [master/b543b351fe95:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-06T05:39:48,512 INFO [master/b543b351fe95:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-06T05:39:48,512 INFO [master/b543b351fe95:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-06T05:39:48,512 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-06T05:39:48,512 INFO [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-06T05:39:48,512 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,38751,1733463581086-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T05:39:48,512 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,38751,1733463581086-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-06T05:39:48,525 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-06T05:39:48,525 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-06T05:39:48,525 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,38751,1733463581086-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:48,558 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-06T05:39:49,561 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38751 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testMasterSessionExpired_1733463585382', {NAME => 'fam', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T05:39:49,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38751 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testMasterSessionExpired_1733463585382 2024-12-06T05:39:49,571 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testMasterSessionExpired_1733463585382 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T05:39:49,571 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:49,571 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38751 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testMasterSessionExpired_1733463585382" procId is: 4 2024-12-06T05:39:49,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38751 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T05:39:49,575 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testMasterSessionExpired_1733463585382 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T05:39:49,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741856_1033 (size=389) 2024-12-06T05:39:49,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741856_1033 (size=389) 2024-12-06T05:39:49,648 INFO [RegionOpenAndInit-testMasterSessionExpired_1733463585382-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f5f7a2a39f20ff5f3d01fd5fd001151c, NAME => 'testMasterSessionExpired_1733463585382,,1733463589561.f5f7a2a39f20ff5f3d01fd5fd001151c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testMasterSessionExpired_1733463585382', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'fam', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:39:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38751 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T05:39:49,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741857_1034 (size=73) 2024-12-06T05:39:49,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741857_1034 (size=73) 2024-12-06T05:39:49,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38751 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T05:39:50,115 DEBUG [RegionOpenAndInit-testMasterSessionExpired_1733463585382-pool-0 {}] regionserver.HRegion(898): Instantiated testMasterSessionExpired_1733463585382,,1733463589561.f5f7a2a39f20ff5f3d01fd5fd001151c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:50,115 DEBUG [RegionOpenAndInit-testMasterSessionExpired_1733463585382-pool-0 {}] regionserver.HRegion(1722): Closing f5f7a2a39f20ff5f3d01fd5fd001151c, disabling compactions & flushes 2024-12-06T05:39:50,115 INFO [RegionOpenAndInit-testMasterSessionExpired_1733463585382-pool-0 {}] regionserver.HRegion(1755): Closing region testMasterSessionExpired_1733463585382,,1733463589561.f5f7a2a39f20ff5f3d01fd5fd001151c. 2024-12-06T05:39:50,115 DEBUG [RegionOpenAndInit-testMasterSessionExpired_1733463585382-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testMasterSessionExpired_1733463585382,,1733463589561.f5f7a2a39f20ff5f3d01fd5fd001151c. 2024-12-06T05:39:50,115 DEBUG [RegionOpenAndInit-testMasterSessionExpired_1733463585382-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testMasterSessionExpired_1733463585382,,1733463589561.f5f7a2a39f20ff5f3d01fd5fd001151c. after waiting 0 ms 2024-12-06T05:39:50,115 DEBUG [RegionOpenAndInit-testMasterSessionExpired_1733463585382-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testMasterSessionExpired_1733463585382,,1733463589561.f5f7a2a39f20ff5f3d01fd5fd001151c. 2024-12-06T05:39:50,116 INFO [RegionOpenAndInit-testMasterSessionExpired_1733463585382-pool-0 {}] regionserver.HRegion(1973): Closed testMasterSessionExpired_1733463585382,,1733463589561.f5f7a2a39f20ff5f3d01fd5fd001151c. 2024-12-06T05:39:50,116 DEBUG [RegionOpenAndInit-testMasterSessionExpired_1733463585382-pool-0 {}] regionserver.HRegion(1676): Region close journal for f5f7a2a39f20ff5f3d01fd5fd001151c: Waiting for close lock at 1733463590115Disabling compacts and flushes for region at 1733463590115Disabling writes for close at 1733463590115Writing region close event to WAL at 1733463590115Closed at 1733463590115 2024-12-06T05:39:50,118 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testMasterSessionExpired_1733463585382 execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T05:39:50,118 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testMasterSessionExpired_1733463585382,,1733463589561.f5f7a2a39f20ff5f3d01fd5fd001151c.","families":{"info":[{"qualifier":"regioninfo","vlen":72,"tag":[],"timestamp":"1733463590118"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733463590118"}]},"ts":"1733463590118"} 2024-12-06T05:39:50,123 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T05:39:50,125 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testMasterSessionExpired_1733463585382 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T05:39:50,125 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testMasterSessionExpired_1733463585382","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733463590125"}]},"ts":"1733463590125"} 2024-12-06T05:39:50,129 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testMasterSessionExpired_1733463585382, state=ENABLING in hbase:meta 2024-12-06T05:39:50,130 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {b543b351fe95=0} racks are {/default-rack=0} 2024-12-06T05:39:50,131 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-06T05:39:50,131 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-06T05:39:50,131 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-06T05:39:50,131 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-06T05:39:50,131 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-06T05:39:50,131 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-06T05:39:50,131 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-06T05:39:50,132 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testMasterSessionExpired_1733463585382, region=f5f7a2a39f20ff5f3d01fd5fd001151c, ASSIGN}] 2024-12-06T05:39:50,134 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testMasterSessionExpired_1733463585382, region=f5f7a2a39f20ff5f3d01fd5fd001151c, ASSIGN 2024-12-06T05:39:50,136 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testMasterSessionExpired_1733463585382, region=f5f7a2a39f20ff5f3d01fd5fd001151c, ASSIGN; state=OFFLINE, location=b543b351fe95,37775,1733463581162; forceNewPlan=false, retain=false 2024-12-06T05:39:50,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38751 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T05:39:50,288 INFO [b543b351fe95:38751 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-06T05:39:50,289 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f5f7a2a39f20ff5f3d01fd5fd001151c, regionState=OPENING, regionLocation=b543b351fe95,37775,1733463581162 2024-12-06T05:39:50,294 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testMasterSessionExpired_1733463585382, region=f5f7a2a39f20ff5f3d01fd5fd001151c, ASSIGN because future has completed 2024-12-06T05:39:50,295 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f7a2a39f20ff5f3d01fd5fd001151c, server=b543b351fe95,37775,1733463581162}] 2024-12-06T05:39:50,456 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T05:39:50,459 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50137, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T05:39:50,466 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testMasterSessionExpired_1733463585382,,1733463589561.f5f7a2a39f20ff5f3d01fd5fd001151c. 2024-12-06T05:39:50,466 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => f5f7a2a39f20ff5f3d01fd5fd001151c, NAME => 'testMasterSessionExpired_1733463585382,,1733463589561.f5f7a2a39f20ff5f3d01fd5fd001151c.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:39:50,467 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMasterSessionExpired_1733463585382 f5f7a2a39f20ff5f3d01fd5fd001151c 2024-12-06T05:39:50,467 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testMasterSessionExpired_1733463585382,,1733463589561.f5f7a2a39f20ff5f3d01fd5fd001151c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:50,467 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for f5f7a2a39f20ff5f3d01fd5fd001151c 2024-12-06T05:39:50,467 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for f5f7a2a39f20ff5f3d01fd5fd001151c 2024-12-06T05:39:50,480 INFO [StoreOpener-f5f7a2a39f20ff5f3d01fd5fd001151c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family fam of region f5f7a2a39f20ff5f3d01fd5fd001151c 2024-12-06T05:39:50,483 INFO [StoreOpener-f5f7a2a39f20ff5f3d01fd5fd001151c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f5f7a2a39f20ff5f3d01fd5fd001151c columnFamilyName fam 2024-12-06T05:39:50,483 DEBUG [StoreOpener-f5f7a2a39f20ff5f3d01fd5fd001151c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:50,483 INFO [StoreOpener-f5f7a2a39f20ff5f3d01fd5fd001151c-1 {}] regionserver.HStore(327): Store=f5f7a2a39f20ff5f3d01fd5fd001151c/fam, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:39:50,484 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for f5f7a2a39f20ff5f3d01fd5fd001151c 2024-12-06T05:39:50,485 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testMasterSessionExpired_1733463585382/f5f7a2a39f20ff5f3d01fd5fd001151c 2024-12-06T05:39:50,486 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testMasterSessionExpired_1733463585382/f5f7a2a39f20ff5f3d01fd5fd001151c 2024-12-06T05:39:50,486 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for f5f7a2a39f20ff5f3d01fd5fd001151c 2024-12-06T05:39:50,487 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for f5f7a2a39f20ff5f3d01fd5fd001151c 2024-12-06T05:39:50,489 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for f5f7a2a39f20ff5f3d01fd5fd001151c 2024-12-06T05:39:50,495 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testMasterSessionExpired_1733463585382/f5f7a2a39f20ff5f3d01fd5fd001151c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:39:50,497 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened f5f7a2a39f20ff5f3d01fd5fd001151c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69398072, jitterRate=0.034111857414245605}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:39:50,497 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f5f7a2a39f20ff5f3d01fd5fd001151c 2024-12-06T05:39:50,498 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for f5f7a2a39f20ff5f3d01fd5fd001151c: Running coprocessor pre-open hook at 1733463590467Writing region info on filesystem at 1733463590467Initializing all the Stores at 1733463590470 (+3 ms)Instantiating store for column family {NAME => 'fam', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463590470Cleaning up temporary data from old regions at 1733463590487 (+17 ms)Running coprocessor post-open hooks at 1733463590497 (+10 ms)Region opened successfully at 1733463590498 (+1 ms) 2024-12-06T05:39:50,499 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testMasterSessionExpired_1733463585382,,1733463589561.f5f7a2a39f20ff5f3d01fd5fd001151c., pid=6, masterSystemTime=1733463590456 2024-12-06T05:39:50,508 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testMasterSessionExpired_1733463585382,,1733463589561.f5f7a2a39f20ff5f3d01fd5fd001151c. 2024-12-06T05:39:50,508 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testMasterSessionExpired_1733463585382,,1733463589561.f5f7a2a39f20ff5f3d01fd5fd001151c. 2024-12-06T05:39:50,509 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f5f7a2a39f20ff5f3d01fd5fd001151c, regionState=OPEN, openSeqNum=2, regionLocation=b543b351fe95,37775,1733463581162 2024-12-06T05:39:50,513 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f7a2a39f20ff5f3d01fd5fd001151c, server=b543b351fe95,37775,1733463581162 because future has completed 2024-12-06T05:39:50,522 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-06T05:39:50,522 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f5f7a2a39f20ff5f3d01fd5fd001151c, server=b543b351fe95,37775,1733463581162 in 224 msec 2024-12-06T05:39:50,529 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-06T05:39:50,529 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testMasterSessionExpired_1733463585382, region=f5f7a2a39f20ff5f3d01fd5fd001151c, ASSIGN in 391 msec 2024-12-06T05:39:50,531 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testMasterSessionExpired_1733463585382 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T05:39:50,531 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testMasterSessionExpired_1733463585382","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733463590531"}]},"ts":"1733463590531"} 2024-12-06T05:39:50,535 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testMasterSessionExpired_1733463585382, state=ENABLED in hbase:meta 2024-12-06T05:39:50,537 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testMasterSessionExpired_1733463585382 execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T05:39:50,541 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testMasterSessionExpired_1733463585382 in 975 msec 2024-12-06T05:39:50,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38751 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T05:39:50,710 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testMasterSessionExpired_1733463585382 completed 2024-12-06T05:39:50,710 INFO [Time-limited test {}] hbase.TestZooKeeper(164): Putting table testMasterSessionExpired_1733463585382 2024-12-06T05:39:50,714 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testMasterSessionExpired_1733463585382', row='testrow', locateType=CURRENT is [region=testMasterSessionExpired_1733463585382,,1733463589561.f5f7a2a39f20ff5f3d01fd5fd001151c., hostname=b543b351fe95,37775,1733463581162, seqNum=2] 2024-12-06T05:39:50,718 DEBUG [Time-limited test {}] master.HMaster(3213): Abort called but aborted=true, stopped=true 2024-12-06T05:39:50,718 ERROR [Time-limited test {}] master.HMaster(3218): Master server abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-06T05:39:50,719 ERROR [Time-limited test {}] master.HMaster(3225): ***** ABORTING master b543b351fe95,38751,1733463581086: killAll ***** 2024-12-06T05:39:50,719 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master 'b543b351fe95,38751,1733463581086' ***** 2024-12-06T05:39:50,719 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: Stopped by Time-limited test 2024-12-06T05:39:50,719 INFO [M:1;b543b351fe95:38751 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T05:39:50,719 INFO [M:1;b543b351fe95:38751 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:1;b543b351fe95:38751. 2024-12-06T05:39:50,719 DEBUG [M:1;b543b351fe95:38751 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T05:39:50,719 DEBUG [M:1;b543b351fe95:38751 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:39:50,719 INFO [M:1;b543b351fe95:38751 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T05:39:50,719 DEBUG [M:1;b543b351fe95:38751 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-06T05:39:50,719 DEBUG [M:1;b543b351fe95:38751 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-06T05:39:50,719 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-06T05:39:50,719 DEBUG [master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.large.0-1733463588464 {}] cleaner.HFileCleaner(306): Exit Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.large.0-1733463588464,5,FailOnTimeoutGroup] 2024-12-06T05:39:50,720 INFO [M:1;b543b351fe95:38751 {}] hbase.ChoreService(370): Chore service for: master/b543b351fe95:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-06T05:39:50,720 INFO [M:1;b543b351fe95:38751 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T05:39:50,720 DEBUG [M:1;b543b351fe95:38751 {}] master.HMaster(1795): Stopping service threads 2024-12-06T05:39:50,720 INFO [M:1;b543b351fe95:38751 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-06T05:39:50,720 INFO [M:1;b543b351fe95:38751 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-06T05:39:50,720 DEBUG [master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.small.0-1733463588469 {}] cleaner.HFileCleaner(306): Exit Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.small.0-1733463588469,5,FailOnTimeoutGroup] 2024-12-06T05:39:50,721 INFO [M:1;b543b351fe95:38751 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=true 2024-12-06T05:39:50,721 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-06T05:39:50,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:50,723 DEBUG [M:1;b543b351fe95:38751 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-12-06T05:39:50,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:50,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37775-0x100ae0a5c730008, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:50,723 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server b543b351fe95,37775,1733463581162: killAll ***** 2024-12-06T05:39:50,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46769-0x100ae0a5c730009, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:50,723 DEBUG [M:1;b543b351fe95:38751 {}] master.ActiveMasterManager(353): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-12-06T05:39:50,723 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-06T05:39:50,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46769-0x100ae0a5c730009, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:50,723 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-06T05:39:50,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37775-0x100ae0a5c730008, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:50,724 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-06T05:39:50,724 INFO [M:1;b543b351fe95:38751 {}] master.ServerManager(1134): Rewriting .lastflushedseqids file at: hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/.lastflushedseqids 2024-12-06T05:39:50,726 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-06T05:39:50,727 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-06T05:39:50,730 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46769-0x100ae0a5c730009, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:39:50,730 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37775-0x100ae0a5c730008, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:39:50,733 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 432607400 }, "NonHeapMemoryUsage": { "committed": 156172288, "init": 7667712, "max": -1, "used": 153698048 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 7149, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 29, "ProcessCallTime_min": 0, "ProcessCallTime_max": 11, "ProcessCallTime_mean": 1, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 1, "ProcessCallTime_75th_percentile": 2, "ProcessCallTime_90th_percentile": 2, "ProcessCallTime_95th_percentile": 11, "ProcessCallTime_98th_percentile": 11, "ProcessCallTime_99th_percentile": 11, "ProcessCallTime_99.9th_percentile": 11, "ProcessCallTime_TimeRangeCount_0-1": 9, "ProcessCallTime_TimeRangeCount_1-3": 2, "ProcessCallTime_TimeRangeCount_10-30": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 29, "QueueCallTime_min": 0, "QueueCallTime_max": 0, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 12, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 29, "TotalCallTime_min": 0, "TotalCallTime_max": 11, "TotalCallTime_mean": 1, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 1, "TotalCallTime_75th_percentile": 2, "TotalCallTime_90th_percentile": 2, "TotalCallTime_95th_percentile": 11, "TotalCallTime_98th_percentile": 11, "TotalCallTime_99th_percentile": 11, "TotalCallTime_99.9th_percentile": 11, "TotalCallTime_TimeRangeCount_0-1": 9, "TotalCallTime_TimeRangeCount_1-3": 2, "TotalCallTime_TimeRangeCount_10-30": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 29, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 51, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 18, "ResponseSize_75th_percentile": 22, "ResponseSize_90th_percentile": 172, "ResponseSize_95th_percentile": 173, "ResponseSize_98th_percentile": 173, "ResponseSize_99th_percentile": 173, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 5, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 3, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 29, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 106, "RequestSize_25th_percentile": 92, "RequestSize_median": 104, "RequestSize_75th_percentile": 119, "RequestSize_90th_percentile": 120, "RequestSize_95th_percentile": 120, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_10-100": 5, "RequestSize_SizeRangeCount_100-1000": 7, "sentBytes": 1935 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:58086", "tag.serverName": "b543b351fe95,37775,1733463581162", "tag.clusterId": "f24feb01-2567-4d5e-a4c6-86f800bb290a", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95", "regionCount": 1, "storeCount": 4, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 74, "memStoreHeapSize": 1248, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733463581162, "averageRegionSize": 74, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 15057, "localBytesRead": 15057, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.4, "writeRequestRatePerSecond": 0.4, "ByteBuffAllocatorHeapAllocationBytes": 15185, "ByteBuffAllocatorPoolAllocationBytes": 332800, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 11, "totalRowActionRequestCount": 9, "readRequestCount": 7, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 2, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 7, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 2, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 7, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 7, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 2, "Put_min": 2, "Put_max": 2, "Put_mean": 2, "Put_25th_percentile": 2, "Put_median": 2, "Put_75th_percentile": 2, "Put_90th_percentile": 2, "Put_95th_percentile": 2, "Put_98th_percentile": 2, "Put_99th_percentile": 2, "Put_99.9th_percentile": 2, "Put_TimeRangeCount_1-3": 2, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 0, "Get_max": 1, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 1, "Get_75th_percentile": 1, "Get_90th_percentile": 1, "Get_95th_percentile": 1, "Get_98th_percentile": 1, "Get_99th_percentile": 1, "Get_99.9th_percentile": 1, "Get_TimeRangeCount_0-1": 2, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 7, "ScanSize_min": 0, "ScanSize_max": 144, "ScanSize_mean": 61, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 144, "ScanSize_90th_percentile": 144, "ScanSize_95th_percentile": 144, "ScanSize_98th_percentile": 144, "ScanSize_99th_percentile": 144, "ScanSize_99.9th_percentile": 144, "ScanSize_SizeRangeCount_0-10": 4, "ScanSize_SizeRangeCount_100-1000": 3, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-06T05:39:50,734 DEBUG [HMaster-EventLoopGroup-8-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b543b351fe95,38751,1733463581086 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:39:50,735 WARN [Time-limited test {}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.regionserver.RegionServerAbortedException): Server b543b351fe95,38751,1733463581086 aborting at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:?] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:?] at org.apache.hadoop.hbase.Abortable.abort(Abortable.java:45) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster.killAll(SingleProcessHBaseCluster.java:880) ~[test-classes/:?] at org.apache.hadoop.hbase.TestZooKeeper.after(TestZooKeeper.java:107) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: Server b543b351fe95,38751,1733463581086 aborting at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-12-06T05:39:50,736 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b543b351fe95,37775,1733463581162' ***** 2024-12-06T05:39:50,736 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: killAll 2024-12-06T05:39:50,737 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server b543b351fe95,46769,1733463581257: killAll ***** 2024-12-06T05:39:50,737 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-06T05:39:50,737 INFO [RS:0;b543b351fe95:37775 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T05:39:50,737 INFO [RS:0;b543b351fe95:37775 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-06T05:39:50,737 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T05:39:50,737 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-06T05:39:50,737 INFO [RS:0;b543b351fe95:37775 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-06T05:39:50,737 INFO [RS:0;b543b351fe95:37775 {}] regionserver.HRegionServer(3091): Received CLOSE for f5f7a2a39f20ff5f3d01fd5fd001151c 2024-12-06T05:39:50,737 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-06T05:39:50,739 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-06T05:39:50,739 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-06T05:39:50,744 INFO [RS:0;b543b351fe95:37775 {}] regionserver.HRegionServer(956): aborting server b543b351fe95,37775,1733463581162 2024-12-06T05:39:50,744 INFO [RS:0;b543b351fe95:37775 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T05:39:50,744 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f5f7a2a39f20ff5f3d01fd5fd001151c, disabling compactions & flushes 2024-12-06T05:39:50,744 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 432607400 }, "NonHeapMemoryUsage": { "committed": 156237824, "init": 7667712, "max": -1, "used": 153775776 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 7149, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 29, "ProcessCallTime_min": 0, "ProcessCallTime_max": 11, "ProcessCallTime_mean": 1, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 1, "ProcessCallTime_75th_percentile": 2, "ProcessCallTime_90th_percentile": 2, "ProcessCallTime_95th_percentile": 11, "ProcessCallTime_98th_percentile": 11, "ProcessCallTime_99th_percentile": 11, "ProcessCallTime_99.9th_percentile": 11, "ProcessCallTime_TimeRangeCount_0-1": 9, "ProcessCallTime_TimeRangeCount_1-3": 2, "ProcessCallTime_TimeRangeCount_10-30": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 29, "QueueCallTime_min": 0, "QueueCallTime_max": 0, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 12, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 29, "TotalCallTime_min": 0, "TotalCallTime_max": 11, "TotalCallTime_mean": 1, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 1, "TotalCallTime_75th_percentile": 2, "TotalCallTime_90th_percentile": 2, "TotalCallTime_95th_percentile": 11, "TotalCallTime_98th_percentile": 11, "TotalCallTime_99th_percentile": 11, "TotalCallTime_99.9th_percentile": 11, "TotalCallTime_TimeRangeCount_0-1": 9, "TotalCallTime_TimeRangeCount_1-3": 2, "TotalCallTime_TimeRangeCount_10-30": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 29, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 51, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 18, "ResponseSize_75th_percentile": 22, "ResponseSize_90th_percentile": 172, "ResponseSize_95th_percentile": 173, "ResponseSize_98th_percentile": 173, "ResponseSize_99th_percentile": 173, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 5, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 3, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 29, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 106, "RequestSize_25th_percentile": 92, "RequestSize_median": 104, "RequestSize_75th_percentile": 119, "RequestSize_90th_percentile": 120, "RequestSize_95th_percentile": 120, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_10-100": 5, "RequestSize_SizeRangeCount_100-1000": 7, "sentBytes": 1935 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:58086", "tag.serverName": "b543b351fe95,37775,1733463581162", "tag.clusterId": "f24feb01-2567-4d5e-a4c6-86f800bb290a", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95", "regionCount": 1, "storeCount": 4, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 74, "memStoreHeapSize": 1248, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733463581162, "averageRegionSize": 74, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 15057, "localBytesRead": 15057, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.4, "writeRequestRatePerSecond": 0.4, "ByteBuffAllocatorHeapAllocationBytes": 15185, "ByteBuffAllocatorPoolAllocationBytes": 332800, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 11, "totalRowActionRequestCount": 9, "readRequestCount": 7, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 2, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 7, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 2, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 7, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 7, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 2, "Put_min": 2, "Put_max": 2, "Put_mean": 2, "Put_25th_percentile": 2, "Put_median": 2, "Put_75th_percentile": 2, "Put_90th_percentile": 2, "Put_95th_percentile": 2, "Put_98th_percentile": 2, "Put_99th_percentile": 2, "Put_99.9th_percentile": 2, "Put_TimeRangeCount_1-3": 2, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 0, "Get_max": 1, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 1, "Get_75th_percentile": 1, "Get_90th_percentile": 1, "Get_95th_percentile": 1, "Get_98th_percentile": 1, "Get_99th_percentile": 1, "Get_99.9th_percentile": 1, "Get_TimeRangeCount_0-1": 2, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 7, "ScanSize_min": 0, "ScanSize_max": 144, "ScanSize_mean": 61, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 144, "ScanSize_90th_percentile": 144, "ScanSize_95th_percentile": 144, "ScanSize_98th_percentile": 144, "ScanSize_99th_percentile": 144, "ScanSize_99.9th_percentile": 144, "ScanSize_SizeRangeCount_0-10": 4, "ScanSize_SizeRangeCount_100-1000": 3, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-06T05:39:50,744 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testMasterSessionExpired_1733463585382,,1733463589561.f5f7a2a39f20ff5f3d01fd5fd001151c. 2024-12-06T05:39:50,744 INFO [RS:0;b543b351fe95:37775 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;b543b351fe95:37775. 2024-12-06T05:39:50,745 DEBUG [RS:0;b543b351fe95:37775 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T05:39:50,745 DEBUG [RS:0;b543b351fe95:37775 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:39:50,745 INFO [RS:0;b543b351fe95:37775 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T05:39:50,745 INFO [RS:0;b543b351fe95:37775 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T05:39:50,745 INFO [RS:0;b543b351fe95:37775 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T05:39:50,745 INFO [RS:0;b543b351fe95:37775 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-06T05:39:50,746 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testMasterSessionExpired_1733463585382,,1733463589561.f5f7a2a39f20ff5f3d01fd5fd001151c. 2024-12-06T05:39:50,746 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testMasterSessionExpired_1733463585382,,1733463589561.f5f7a2a39f20ff5f3d01fd5fd001151c. after waiting 0 ms 2024-12-06T05:39:50,746 DEBUG [HMaster-EventLoopGroup-8-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b543b351fe95,38751,1733463581086 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:39:50,746 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testMasterSessionExpired_1733463585382,,1733463589561.f5f7a2a39f20ff5f3d01fd5fd001151c. 2024-12-06T05:39:50,746 WARN [Time-limited test {}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.regionserver.RegionServerAbortedException): Server b543b351fe95,38751,1733463581086 aborting at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:?] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:?] at org.apache.hadoop.hbase.Abortable.abort(Abortable.java:45) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster.killAll(SingleProcessHBaseCluster.java:880) ~[test-classes/:?] at org.apache.hadoop.hbase.TestZooKeeper.after(TestZooKeeper.java:107) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: Server b543b351fe95,38751,1733463581086 aborting at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-12-06T05:39:50,749 INFO [RS:0;b543b351fe95:37775 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-06T05:39:50,749 DEBUG [RS:0;b543b351fe95:37775 {}] regionserver.HRegionServer(1325): Online Regions={f5f7a2a39f20ff5f3d01fd5fd001151c=testMasterSessionExpired_1733463585382,,1733463589561.f5f7a2a39f20ff5f3d01fd5fd001151c., 1588230740=hbase:meta,,1.1588230740} 2024-12-06T05:39:50,749 DEBUG [RS:0;b543b351fe95:37775 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, f5f7a2a39f20ff5f3d01fd5fd001151c 2024-12-06T05:39:50,750 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T05:39:50,750 INFO [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T05:39:50,750 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T05:39:50,750 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T05:39:50,751 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T05:39:50,753 ERROR [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1960): Memstore data size is 41 in region testMasterSessionExpired_1733463585382,,1733463589561.f5f7a2a39f20ff5f3d01fd5fd001151c. 2024-12-06T05:39:50,754 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testMasterSessionExpired_1733463585382,,1733463589561.f5f7a2a39f20ff5f3d01fd5fd001151c. 2024-12-06T05:39:50,754 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f5f7a2a39f20ff5f3d01fd5fd001151c: Waiting for close lock at 1733463590744Running coprocessor pre-close hooks at 1733463590744Disabling compacts and flushes for region at 1733463590744Disabling writes for close at 1733463590746 (+2 ms)Writing region close event to WAL at 1733463590753 (+7 ms)Running coprocessor post-close hooks at 1733463590753Closed at 1733463590754 (+1 ms) 2024-12-06T05:39:50,754 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testMasterSessionExpired_1733463585382,,1733463589561.f5f7a2a39f20ff5f3d01fd5fd001151c. 2024-12-06T05:39:50,757 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b543b351fe95,46769,1733463581257' ***** 2024-12-06T05:39:50,757 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: killAll 2024-12-06T05:39:50,757 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T05:39:50,757 INFO [RS:1;b543b351fe95:46769 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T05:39:50,758 INFO [RS:1;b543b351fe95:46769 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-06T05:39:50,758 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.TestZooKeeper.after(TestZooKeeper.java:110) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T05:39:50,758 INFO [RS:1;b543b351fe95:46769 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-06T05:39:50,758 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:39:50,758 INFO [RS:1;b543b351fe95:46769 {}] regionserver.HRegionServer(956): aborting server b543b351fe95,46769,1733463581257 2024-12-06T05:39:50,758 INFO [RS:1;b543b351fe95:46769 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T05:39:50,758 INFO [RS:1;b543b351fe95:46769 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;b543b351fe95:46769. 2024-12-06T05:39:50,758 DEBUG [RS:1;b543b351fe95:46769 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T05:39:50,758 DEBUG [RS:1;b543b351fe95:46769 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:39:50,758 INFO [RS:1;b543b351fe95:46769 {}] regionserver.HRegionServer(976): stopping server b543b351fe95,46769,1733463581257; all regions closed. 2024-12-06T05:39:50,758 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2602908d to 127.0.0.1:58086 2024-12-06T05:39:50,758 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-06T05:39:50,758 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T05:39:50,763 ERROR [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 1707 in region hbase:meta,,1.1588230740 2024-12-06T05:39:50,763 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-06T05:39:50,763 INFO [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T05:39:50,763 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733463590750Running coprocessor pre-close hooks at 1733463590750Disabling compacts and flushes for region at 1733463590750Disabling writes for close at 1733463590750Writing region close event to WAL at 1733463590763 (+13 ms)Running coprocessor post-close hooks at 1733463590763Closed at 1733463590763 2024-12-06T05:39:50,764 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-06T05:39:50,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741847_1023 (size=93) 2024-12-06T05:39:50,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741847_1023 (size=93) 2024-12-06T05:39:50,773 DEBUG [RS:1;b543b351fe95:46769 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:39:50,773 INFO [RS:1;b543b351fe95:46769 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T05:39:50,773 INFO [RS:1;b543b351fe95:46769 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T05:39:50,773 INFO [RS:1;b543b351fe95:46769 {}] hbase.ChoreService(370): Chore service for: regionserver/b543b351fe95:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-06T05:39:50,773 INFO [RS:1;b543b351fe95:46769 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T05:39:50,774 INFO [regionserver/b543b351fe95:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T05:39:50,774 INFO [RS:1;b543b351fe95:46769 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T05:39:50,774 INFO [RS:1;b543b351fe95:46769 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T05:39:50,774 INFO [RS:1;b543b351fe95:46769 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T05:39:50,774 INFO [RS:1;b543b351fe95:46769 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46769 2024-12-06T05:39:50,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46769-0x100ae0a5c730009, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b543b351fe95,46769,1733463581257 2024-12-06T05:39:50,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T05:39:50,775 INFO [RS:1;b543b351fe95:46769 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T05:39:50,785 INFO [regionserver/b543b351fe95:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T05:39:50,787 INFO [regionserver/b543b351fe95:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T05:39:50,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741858_1035 (size=147) 2024-12-06T05:39:50,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741858_1035 (size=147) 2024-12-06T05:39:50,820 INFO [M:1;b543b351fe95:38751 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-06T05:39:50,821 INFO [M:1;b543b351fe95:38751 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=true 2024-12-06T05:39:50,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741852_1029 (size=17508) 2024-12-06T05:39:50,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741852_1029 (size=17508) 2024-12-06T05:39:50,853 DEBUG [M:1;b543b351fe95:38751 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T05:39:50,853 INFO [M:1;b543b351fe95:38751 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:50,853 DEBUG [M:1;b543b351fe95:38751 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:50,853 DEBUG [M:1;b543b351fe95:38751 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T05:39:50,853 DEBUG [M:1;b543b351fe95:38751 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:50,865 ERROR [M:1;b543b351fe95:38751 {}] regionserver.HRegion(1960): Memstore data size is 15671 in region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:50,865 INFO [M:1;b543b351fe95:38751 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:50,865 DEBUG [M:1;b543b351fe95:38751 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733463590853Disabling compacts and flushes for region at 1733463590853Disabling writes for close at 1733463590853Writing region close event to WAL at 1733463590865 (+12 ms)Closed at 1733463590865 2024-12-06T05:39:50,865 INFO [M:1;b543b351fe95:38751 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-06T05:39:50,865 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T05:39:50,865 INFO [M:1;b543b351fe95:38751 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38751 2024-12-06T05:39:50,866 INFO [M:1;b543b351fe95:38751 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T05:39:50,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46769-0x100ae0a5c730009, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:39:50,876 INFO [RS:1;b543b351fe95:46769 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T05:39:50,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46769-0x100ae0a5c730009, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:39:50,876 INFO [RS:1;b543b351fe95:46769 {}] regionserver.HRegionServer(1031): Exiting; stopping=b543b351fe95,46769,1733463581257; zookeeper connection closed. 2024-12-06T05:39:50,876 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@8a55be6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@8a55be6 2024-12-06T05:39:50,949 INFO [RS:0;b543b351fe95:37775 {}] regionserver.HRegionServer(976): stopping server b543b351fe95,37775,1733463581162; all regions closed. 2024-12-06T05:39:50,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741849_1025 (size=2349) 2024-12-06T05:39:50,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741849_1025 (size=2349) 2024-12-06T05:39:50,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741848_1024 (size=583) 2024-12-06T05:39:50,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741848_1024 (size=583) 2024-12-06T05:39:50,957 DEBUG [RS:0;b543b351fe95:37775 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:39:50,957 INFO [RS:0;b543b351fe95:37775 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T05:39:50,957 INFO [RS:0;b543b351fe95:37775 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T05:39:50,957 INFO [RS:0;b543b351fe95:37775 {}] hbase.ChoreService(370): Chore service for: regionserver/b543b351fe95:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-06T05:39:50,958 INFO [RS:0;b543b351fe95:37775 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T05:39:50,958 INFO [regionserver/b543b351fe95:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T05:39:50,958 INFO [RS:0;b543b351fe95:37775 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37775 2024-12-06T05:39:50,961 INFO [RS:0;b543b351fe95:37775 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T05:39:50,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37775-0x100ae0a5c730008, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b543b351fe95,37775,1733463581162 2024-12-06T05:39:50,967 INFO [M:1;b543b351fe95:38751 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T05:39:50,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:39:50,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38751-0x100ae0a5c730007, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:39:51,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37775-0x100ae0a5c730008, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:39:51,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37775-0x100ae0a5c730008, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:39:51,062 INFO [RS:0;b543b351fe95:37775 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T05:39:51,062 INFO [RS:0;b543b351fe95:37775 {}] regionserver.HRegionServer(1031): Exiting; stopping=b543b351fe95,37775,1733463581162; zookeeper connection closed. 2024-12-06T05:39:51,062 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6aee98f4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6aee98f4 2024-12-06T05:39:51,063 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 2 master(s) and 2 regionserver(s) complete 2024-12-06T05:39:51,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730005, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:39:51,164 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:51,169 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=testing utility connecting to ZooKeeper ensemble=127.0.0.1:58086 2024-12-06T05:39:51,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility0x0, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:39:51,171 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): testing utility-0x100ae0a5c73000b connected 2024-12-06T05:39:51,175 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-06T05:39:51,175 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/meta-region-server 2024-12-06T05:39:51,176 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/rs 2024-12-06T05:39:51,177 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/splitWAL 2024-12-06T05:39:51,177 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters 2024-12-06T05:39:51,178 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc 2024-12-06T05:39:51,178 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/draining 2024-12-06T05:39:51,178 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/master-maintenance 2024-12-06T05:39:51,183 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot 2024-12-06T05:39:51,184 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/hbaseid 2024-12-06T05:39:51,184 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/table 2024-12-06T05:39:51,187 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc/abort 2024-12-06T05:39:51,188 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc/acquired 2024-12-06T05:39:51,188 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc/reached 2024-12-06T05:39:51,190 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot/abort 2024-12-06T05:39:51,191 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot/acquired 2024-12-06T05:39:51,192 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot/reached 2024-12-06T05:39:51,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/reached 2024-12-06T05:39:51,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/online-snapshot 2024-12-06T05:39:51,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/acquired 2024-12-06T05:39:51,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/abort 2024-12-06T05:39:51,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/reached 2024-12-06T05:39:51,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/flush-table-proc 2024-12-06T05:39:51,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/acquired 2024-12-06T05:39:51,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/abort 2024-12-06T05:39:51,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/table 2024-12-06T05:39:51,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:51,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/hbaseid 2024-12-06T05:39:51,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot 2024-12-06T05:39:51,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master-maintenance 2024-12-06T05:39:51,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/draining 2024-12-06T05:39:51,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc 2024-12-06T05:39:51,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:51,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/splitWAL 2024-12-06T05:39:51,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs 2024-12-06T05:39:51,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:39:51,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T05:39:51,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase 2024-12-06T05:39:51,213 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestZooKeeper#testMasterSessionExpired Thread=319 (was 289) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-10-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (268780456) connection to localhost/127.0.0.1:42031 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (268780456) connection to localhost/127.0.0.1:34453 from jenkins.hfs.2 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-10-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/cluster_05577cd5-3697-aeeb-f42f-dcca3f191df1/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/cluster_05577cd5-3697-aeeb-f42f-dcca3f191df1/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/cluster_05577cd5-3697-aeeb-f42f-dcca3f191df1/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/cluster_05577cd5-3697-aeeb-f42f-dcca3f191df1/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1471901137_22 at /127.0.0.1:35784 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (268780456) connection to localhost/127.0.0.1:34453 from jenkins.hfs.3 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:34453 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-10-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_351197811_22 at /127.0.0.1:58202 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:34453 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42031 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=592 (was 551) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=408 (was 303) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4642 (was 5101) 2024-12-06T05:39:51,224 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestZooKeeper#testMasterZKSessionRecoveryFailure Thread=319, OpenFileDescriptor=592, MaxFileDescriptor=1048576, SystemLoadAverage=408, ProcessCount=11, AvailableMemoryMB=4642 2024-12-06T05:39:51,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741859_1036 (size=7) 2024-12-06T05:39:51,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741859_1036 (size=7) 2024-12-06T05:39:51,655 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 with version=8 2024-12-06T05:39:51,656 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/hbase-staging 2024-12-06T05:39:51,658 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b543b351fe95:0 server-side Connection retries=45 2024-12-06T05:39:51,658 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:51,658 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:51,659 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T05:39:51,659 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:51,659 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T05:39:51,659 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-06T05:39:51,659 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T05:39:51,666 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39613 2024-12-06T05:39:51,668 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39613 connecting to ZooKeeper ensemble=127.0.0.1:58086 2024-12-06T05:39:51,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:396130x0, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:39:51,672 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39613-0x100ae0a5c73000c connected 2024-12-06T05:39:51,684 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:51,686 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:51,689 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39613-0x100ae0a5c73000c, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T05:39:51,689 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279, hbase.cluster.distributed=false 2024-12-06T05:39:51,691 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39613-0x100ae0a5c73000c, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T05:39:51,691 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39613 2024-12-06T05:39:51,691 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39613 2024-12-06T05:39:51,693 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39613 2024-12-06T05:39:51,712 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39613 2024-12-06T05:39:51,717 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39613 2024-12-06T05:39:51,719 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b543b351fe95:0 server-side Connection retries=45 2024-12-06T05:39:51,719 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:51,719 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:51,719 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T05:39:51,719 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:51,719 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T05:39:51,719 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-06T05:39:51,719 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T05:39:51,720 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40467 2024-12-06T05:39:51,722 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40467 connecting to ZooKeeper ensemble=127.0.0.1:58086 2024-12-06T05:39:51,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:404670x0, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:39:51,725 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40467-0x100ae0a5c73000d connected 2024-12-06T05:39:51,729 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:51,730 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:51,732 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T05:39:51,732 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279, hbase.cluster.distributed=false 2024-12-06T05:39:51,733 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T05:39:51,741 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40467 2024-12-06T05:39:51,741 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40467 2024-12-06T05:39:51,743 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40467 2024-12-06T05:39:51,743 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40467 2024-12-06T05:39:51,744 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40467 2024-12-06T05:39:51,764 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b543b351fe95:0 server-side Connection retries=45 2024-12-06T05:39:51,764 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:51,765 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:51,765 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T05:39:51,765 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:51,765 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T05:39:51,765 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T05:39:51,765 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T05:39:51,766 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37213 2024-12-06T05:39:51,768 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37213 connecting to ZooKeeper ensemble=127.0.0.1:58086 2024-12-06T05:39:51,768 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:51,771 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:51,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:372130x0, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:39:51,777 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37213-0x100ae0a5c73000e connected 2024-12-06T05:39:51,777 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37213-0x100ae0a5c73000e, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T05:39:51,777 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T05:39:51,780 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T05:39:51,780 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37213-0x100ae0a5c73000e, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:39:51,785 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37213-0x100ae0a5c73000e, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T05:39:51,786 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37213 2024-12-06T05:39:51,787 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37213 2024-12-06T05:39:51,787 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37213 2024-12-06T05:39:51,787 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37213 2024-12-06T05:39:51,787 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37213 2024-12-06T05:39:51,803 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b543b351fe95:0 server-side Connection retries=45 2024-12-06T05:39:51,803 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:51,803 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:51,803 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T05:39:51,803 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:51,803 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T05:39:51,803 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T05:39:51,804 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T05:39:51,805 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37497 2024-12-06T05:39:51,807 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37497 connecting to ZooKeeper ensemble=127.0.0.1:58086 2024-12-06T05:39:51,808 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:51,810 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:51,818 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:374970x0, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T05:39:51,818 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T05:39:51,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:374970x0, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:39:51,837 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T05:39:51,838 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37497-0x100ae0a5c73000f connected 2024-12-06T05:39:51,839 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37497-0x100ae0a5c73000f, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:39:51,840 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37497-0x100ae0a5c73000f, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T05:39:51,849 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37497 2024-12-06T05:39:51,861 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37497 2024-12-06T05:39:51,864 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37497 2024-12-06T05:39:51,865 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37497 2024-12-06T05:39:51,865 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37497 2024-12-06T05:39:51,886 DEBUG [M:0;b543b351fe95:39613 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b543b351fe95:39613 2024-12-06T05:39:51,894 DEBUG [M:1;b543b351fe95:40467 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:1;b543b351fe95:40467 2024-12-06T05:39:51,895 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b543b351fe95,39613,1733463591658 2024-12-06T05:39:51,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:51,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39613-0x100ae0a5c73000c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:51,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37497-0x100ae0a5c73000f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:51,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37213-0x100ae0a5c73000e, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:51,900 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b543b351fe95,40467,1733463591718 2024-12-06T05:39:51,900 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39613-0x100ae0a5c73000c, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b543b351fe95,39613,1733463591658 2024-12-06T05:39:51,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39613-0x100ae0a5c73000c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:51,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37213-0x100ae0a5c73000e, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:51,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:51,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:51,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39613-0x100ae0a5c73000c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:51,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37213-0x100ae0a5c73000e, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:51,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37213-0x100ae0a5c73000e, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:51,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37497-0x100ae0a5c73000f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:51,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37497-0x100ae0a5c73000f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:51,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37497-0x100ae0a5c73000f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:51,905 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b543b351fe95,40467,1733463591718 2024-12-06T05:39:51,907 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39613-0x100ae0a5c73000c, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T05:39:51,910 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b543b351fe95,39613,1733463591658 from backup master directory 2024-12-06T05:39:51,910 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T05:39:51,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:51,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37213-0x100ae0a5c73000e, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:51,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37497-0x100ae0a5c73000f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:51,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39613-0x100ae0a5c73000c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b543b351fe95,39613,1733463591658 2024-12-06T05:39:51,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39613-0x100ae0a5c73000c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:51,914 WARN [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T05:39:51,914 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b543b351fe95,39613,1733463591658 2024-12-06T05:39:51,915 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ActiveMasterManager(296): Another master is the active master, b543b351fe95,39613,1733463591658; waiting to become the next active master 2024-12-06T05:39:51,923 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/hbase.id] with ID: 72da7897-65c7-419a-97b9-fb2a1bc3f93c 2024-12-06T05:39:51,923 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/.tmp/hbase.id 2024-12-06T05:39:51,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741860_1037 (size=42) 2024-12-06T05:39:51,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741860_1037 (size=42) 2024-12-06T05:39:51,938 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/.tmp/hbase.id]:[hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/hbase.id] 2024-12-06T05:39:51,956 INFO [master/b543b351fe95:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:51,956 INFO [master/b543b351fe95:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-06T05:39:51,958 INFO [master/b543b351fe95:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-12-06T05:39:51,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:51,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37497-0x100ae0a5c73000f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:51,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37213-0x100ae0a5c73000e, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:51,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39613-0x100ae0a5c73000c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:51,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741861_1038 (size=196) 2024-12-06T05:39:51,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741861_1038 (size=196) 2024-12-06T05:39:51,988 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T05:39:51,989 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-06T05:39:51,989 INFO [master/b543b351fe95:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:39:52,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741862_1039 (size=1189) 2024-12-06T05:39:52,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741862_1039 (size=1189) 2024-12-06T05:39:52,014 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store 2024-12-06T05:39:52,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741863_1040 (size=34) 2024-12-06T05:39:52,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741863_1040 (size=34) 2024-12-06T05:39:52,044 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:52,044 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T05:39:52,045 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:52,045 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:52,045 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T05:39:52,045 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:52,045 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:52,045 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733463592044Disabling compacts and flushes for region at 1733463592044Disabling writes for close at 1733463592045 (+1 ms)Writing region close event to WAL at 1733463592045Closed at 1733463592045 2024-12-06T05:39:52,046 WARN [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/.initializing 2024-12-06T05:39:52,046 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,39613,1733463591658 2024-12-06T05:39:52,047 INFO [master/b543b351fe95:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:39:52,051 INFO [master/b543b351fe95:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b543b351fe95%2C39613%2C1733463591658, suffix=, logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,39613,1733463591658, archiveDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/oldWALs, maxLogs=10 2024-12-06T05:39:52,068 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,39613,1733463591658/b543b351fe95%2C39613%2C1733463591658.1733463592051, exclude list is [], retry=0 2024-12-06T05:39:52,075 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39561,DS-4485eaa6-367b-4084-97e1-b834fb8d8596,DISK] 2024-12-06T05:39:52,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-80ddb16c-ec66-4e39-a44c-0a80fa143251,DISK] 2024-12-06T05:39:52,090 INFO [master/b543b351fe95:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,39613,1733463591658/b543b351fe95%2C39613%2C1733463591658.1733463592051 2024-12-06T05:39:52,093 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43559:43559),(127.0.0.1/127.0.0.1:44855:44855)] 2024-12-06T05:39:52,093 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:39:52,093 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:52,094 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:52,094 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:52,097 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:52,105 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-06T05:39:52,106 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:52,106 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:52,107 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:52,109 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-06T05:39:52,109 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:52,110 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:39:52,110 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:52,112 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-06T05:39:52,113 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:52,113 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:39:52,114 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:52,116 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-06T05:39:52,116 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:52,117 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:39:52,118 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:52,119 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:52,120 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:52,124 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:52,124 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:52,124 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T05:39:52,126 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:52,134 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:39:52,135 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70555090, jitterRate=0.05135276913642883}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T05:39:52,136 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733463592094Initializing all the Stores at 1733463592095 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463592095Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463592097 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463592097Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463592097Cleaning up temporary data from old regions at 1733463592124 (+27 ms)Region opened successfully at 1733463592135 (+11 ms) 2024-12-06T05:39:52,136 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-06T05:39:52,142 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6be4b857, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b543b351fe95/172.17.0.2:0 2024-12-06T05:39:52,143 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-06T05:39:52,143 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-06T05:39:52,144 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-06T05:39:52,144 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-06T05:39:52,144 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-06T05:39:52,145 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-06T05:39:52,145 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-06T05:39:52,148 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-06T05:39:52,149 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39613-0x100ae0a5c73000c, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-06T05:39:52,151 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-06T05:39:52,151 INFO [master/b543b351fe95:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-06T05:39:52,152 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39613-0x100ae0a5c73000c, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-06T05:39:52,152 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-06T05:39:52,153 INFO [master/b543b351fe95:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-06T05:39:52,154 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39613-0x100ae0a5c73000c, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-06T05:39:52,154 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-06T05:39:52,155 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39613-0x100ae0a5c73000c, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-06T05:39:52,157 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-06T05:39:52,159 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39613-0x100ae0a5c73000c, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-06T05:39:52,159 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-06T05:39:52,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39613-0x100ae0a5c73000c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T05:39:52,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37497-0x100ae0a5c73000f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T05:39:52,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T05:39:52,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:52,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37213-0x100ae0a5c73000e, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T05:39:52,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37213-0x100ae0a5c73000e, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:52,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37497-0x100ae0a5c73000f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:52,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39613-0x100ae0a5c73000c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:52,162 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=b543b351fe95,39613,1733463591658, sessionid=0x100ae0a5c73000c, setting cluster-up flag (Was=false) 2024-12-06T05:39:52,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37213-0x100ae0a5c73000e, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:52,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39613-0x100ae0a5c73000c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:52,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37497-0x100ae0a5c73000f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:52,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:52,169 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-06T05:39:52,172 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b543b351fe95,39613,1733463591658 2024-12-06T05:39:52,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37497-0x100ae0a5c73000f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:52,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:52,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39613-0x100ae0a5c73000c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:52,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37213-0x100ae0a5c73000e, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:52,179 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-06T05:39:52,181 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b543b351fe95,39613,1733463591658 2024-12-06T05:39:52,183 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-06T05:39:52,185 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-06T05:39:52,186 INFO [master/b543b351fe95:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-06T05:39:52,186 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b543b351fe95,39613,1733463591658 Number of backup masters: 1 b543b351fe95,40467,1733463591718 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-06T05:39:52,194 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:39:52,194 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:39:52,194 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:39:52,194 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:39:52,194 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b543b351fe95:0, corePoolSize=10, maxPoolSize=10 2024-12-06T05:39:52,194 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:52,194 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b543b351fe95:0, corePoolSize=2, maxPoolSize=2 2024-12-06T05:39:52,195 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:52,199 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733463622199 2024-12-06T05:39:52,199 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-06T05:39:52,199 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T05:39:52,199 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-06T05:39:52,199 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-06T05:39:52,199 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-06T05:39:52,199 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-06T05:39:52,199 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-06T05:39:52,200 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-06T05:39:52,201 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:52,201 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-06T05:39:52,204 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:52,207 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-06T05:39:52,208 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-06T05:39:52,208 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-06T05:39:52,224 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-06T05:39:52,224 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-06T05:39:52,225 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.large.0-1733463592224,5,FailOnTimeoutGroup] 2024-12-06T05:39:52,237 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.small.0-1733463592225,5,FailOnTimeoutGroup] 2024-12-06T05:39:52,237 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:52,237 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-06T05:39:52,237 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:52,237 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:52,285 INFO [RS:0;b543b351fe95:37213 {}] regionserver.HRegionServer(746): ClusterId : 72da7897-65c7-419a-97b9-fb2a1bc3f93c 2024-12-06T05:39:52,285 DEBUG [RS:0;b543b351fe95:37213 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T05:39:52,289 DEBUG [RS:0;b543b351fe95:37213 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T05:39:52,289 DEBUG [RS:0;b543b351fe95:37213 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T05:39:52,289 INFO [RS:1;b543b351fe95:37497 {}] regionserver.HRegionServer(746): ClusterId : 72da7897-65c7-419a-97b9-fb2a1bc3f93c 2024-12-06T05:39:52,289 DEBUG [RS:1;b543b351fe95:37497 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T05:39:52,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741865_1042 (size=1321) 2024-12-06T05:39:52,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741865_1042 (size=1321) 2024-12-06T05:39:52,293 DEBUG [RS:0;b543b351fe95:37213 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T05:39:52,294 DEBUG [RS:0;b543b351fe95:37213 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@445240c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b543b351fe95/172.17.0.2:0 2024-12-06T05:39:52,294 DEBUG [RS:1;b543b351fe95:37497 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T05:39:52,295 DEBUG [RS:1;b543b351fe95:37497 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T05:39:52,306 DEBUG [RS:1;b543b351fe95:37497 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T05:39:52,307 DEBUG [RS:1;b543b351fe95:37497 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@287608c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b543b351fe95/172.17.0.2:0 2024-12-06T05:39:52,316 DEBUG [RS:0;b543b351fe95:37213 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b543b351fe95:37213 2024-12-06T05:39:52,316 INFO [RS:0;b543b351fe95:37213 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T05:39:52,316 INFO [RS:0;b543b351fe95:37213 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T05:39:52,316 DEBUG [RS:0;b543b351fe95:37213 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T05:39:52,317 INFO [RS:0;b543b351fe95:37213 {}] regionserver.HRegionServer(2659): reportForDuty to master=b543b351fe95,39613,1733463591658 with port=37213, startcode=1733463591764 2024-12-06T05:39:52,317 DEBUG [RS:0;b543b351fe95:37213 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T05:39:52,330 INFO [HMaster-EventLoopGroup-11-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39491, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T05:39:52,330 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39613 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b543b351fe95,37213,1733463591764 2024-12-06T05:39:52,331 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39613 {}] master.ServerManager(517): Registering regionserver=b543b351fe95,37213,1733463591764 2024-12-06T05:39:52,334 DEBUG [RS:1;b543b351fe95:37497 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;b543b351fe95:37497 2024-12-06T05:39:52,336 INFO [RS:1;b543b351fe95:37497 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T05:39:52,336 INFO [RS:1;b543b351fe95:37497 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T05:39:52,336 DEBUG [RS:1;b543b351fe95:37497 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T05:39:52,337 INFO [RS:1;b543b351fe95:37497 {}] regionserver.HRegionServer(2659): reportForDuty to master=b543b351fe95,39613,1733463591658 with port=37497, startcode=1733463591802 2024-12-06T05:39:52,337 DEBUG [RS:1;b543b351fe95:37497 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T05:39:52,339 INFO [HMaster-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44001, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T05:39:52,340 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39613 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b543b351fe95,37497,1733463591802 2024-12-06T05:39:52,340 DEBUG [RS:0;b543b351fe95:37213 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:39:52,340 DEBUG [RS:0;b543b351fe95:37213 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34453 2024-12-06T05:39:52,341 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39613 {}] master.ServerManager(517): Registering regionserver=b543b351fe95,37497,1733463591802 2024-12-06T05:39:52,341 DEBUG [RS:0;b543b351fe95:37213 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T05:39:52,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39613-0x100ae0a5c73000c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T05:39:52,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T05:39:52,343 DEBUG [RS:0;b543b351fe95:37213 {}] zookeeper.ZKUtil(111): regionserver:37213-0x100ae0a5c73000e, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b543b351fe95,37213,1733463591764 2024-12-06T05:39:52,343 WARN [RS:0;b543b351fe95:37213 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T05:39:52,343 INFO [RS:0;b543b351fe95:37213 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:39:52,344 DEBUG [RS:0;b543b351fe95:37213 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,37213,1733463591764 2024-12-06T05:39:52,344 DEBUG [RS:1;b543b351fe95:37497 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:39:52,344 DEBUG [RS:1;b543b351fe95:37497 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34453 2024-12-06T05:39:52,344 DEBUG [RS:1;b543b351fe95:37497 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T05:39:52,349 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b543b351fe95,37213,1733463591764] 2024-12-06T05:39:52,349 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b543b351fe95,37497,1733463591802] 2024-12-06T05:39:52,350 DEBUG [RS:1;b543b351fe95:37497 {}] zookeeper.ZKUtil(111): regionserver:37497-0x100ae0a5c73000f, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b543b351fe95,37497,1733463591802 2024-12-06T05:39:52,350 WARN [RS:1;b543b351fe95:37497 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T05:39:52,350 INFO [RS:1;b543b351fe95:37497 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:39:52,350 DEBUG [RS:1;b543b351fe95:37497 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,37497,1733463591802 2024-12-06T05:39:52,378 INFO [RS:0;b543b351fe95:37213 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T05:39:52,391 INFO [RS:0;b543b351fe95:37213 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T05:39:52,399 INFO [RS:0;b543b351fe95:37213 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T05:39:52,399 INFO [RS:0;b543b351fe95:37213 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:52,412 INFO [RS:0;b543b351fe95:37213 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T05:39:52,413 INFO [RS:0;b543b351fe95:37213 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T05:39:52,413 INFO [RS:0;b543b351fe95:37213 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:52,414 DEBUG [RS:0;b543b351fe95:37213 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:52,414 DEBUG [RS:0;b543b351fe95:37213 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:52,414 DEBUG [RS:0;b543b351fe95:37213 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:52,414 DEBUG [RS:0;b543b351fe95:37213 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:52,414 DEBUG [RS:0;b543b351fe95:37213 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:52,414 DEBUG [RS:0;b543b351fe95:37213 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0, corePoolSize=2, maxPoolSize=2 2024-12-06T05:39:52,414 DEBUG [RS:0;b543b351fe95:37213 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:52,414 DEBUG [RS:0;b543b351fe95:37213 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:52,414 DEBUG [RS:0;b543b351fe95:37213 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:52,414 DEBUG [RS:0;b543b351fe95:37213 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:52,414 DEBUG [RS:0;b543b351fe95:37213 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:52,414 DEBUG [RS:0;b543b351fe95:37213 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:52,415 DEBUG [RS:0;b543b351fe95:37213 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b543b351fe95:0, corePoolSize=3, maxPoolSize=3 2024-12-06T05:39:52,415 DEBUG [RS:0;b543b351fe95:37213 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b543b351fe95:0, corePoolSize=3, maxPoolSize=3 2024-12-06T05:39:52,425 INFO [RS:1;b543b351fe95:37497 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T05:39:52,427 INFO [RS:0;b543b351fe95:37213 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:52,427 INFO [RS:0;b543b351fe95:37213 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:52,428 INFO [RS:0;b543b351fe95:37213 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:52,428 INFO [RS:0;b543b351fe95:37213 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:52,428 INFO [RS:0;b543b351fe95:37213 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:52,428 INFO [RS:0;b543b351fe95:37213 {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,37213,1733463591764-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T05:39:52,429 INFO [RS:1;b543b351fe95:37497 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T05:39:52,445 INFO [RS:1;b543b351fe95:37497 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T05:39:52,445 INFO [RS:1;b543b351fe95:37497 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:52,462 INFO [RS:1;b543b351fe95:37497 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T05:39:52,463 INFO [RS:1;b543b351fe95:37497 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T05:39:52,463 INFO [RS:1;b543b351fe95:37497 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:52,463 DEBUG [RS:1;b543b351fe95:37497 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:52,463 DEBUG [RS:1;b543b351fe95:37497 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:52,463 DEBUG [RS:1;b543b351fe95:37497 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:52,463 DEBUG [RS:1;b543b351fe95:37497 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:52,464 DEBUG [RS:1;b543b351fe95:37497 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:52,465 DEBUG [RS:1;b543b351fe95:37497 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0, corePoolSize=2, maxPoolSize=2 2024-12-06T05:39:52,466 DEBUG [RS:1;b543b351fe95:37497 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:52,466 DEBUG [RS:1;b543b351fe95:37497 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:52,466 DEBUG [RS:1;b543b351fe95:37497 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:52,466 DEBUG [RS:1;b543b351fe95:37497 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:52,466 DEBUG [RS:1;b543b351fe95:37497 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:52,466 DEBUG [RS:1;b543b351fe95:37497 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:52,466 DEBUG [RS:1;b543b351fe95:37497 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b543b351fe95:0, corePoolSize=3, maxPoolSize=3 2024-12-06T05:39:52,466 DEBUG [RS:1;b543b351fe95:37497 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b543b351fe95:0, corePoolSize=3, maxPoolSize=3 2024-12-06T05:39:52,468 INFO [RS:0;b543b351fe95:37213 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T05:39:52,468 INFO [RS:0;b543b351fe95:37213 {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,37213,1733463591764-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:52,468 INFO [RS:0;b543b351fe95:37213 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:52,468 INFO [RS:0;b543b351fe95:37213 {}] regionserver.Replication(171): b543b351fe95,37213,1733463591764 started 2024-12-06T05:39:52,474 INFO [RS:1;b543b351fe95:37497 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:52,475 INFO [RS:1;b543b351fe95:37497 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:52,475 INFO [RS:1;b543b351fe95:37497 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:52,475 INFO [RS:1;b543b351fe95:37497 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:52,475 INFO [RS:1;b543b351fe95:37497 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:52,475 INFO [RS:1;b543b351fe95:37497 {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,37497,1733463591802-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T05:39:52,495 INFO [RS:0;b543b351fe95:37213 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:52,496 INFO [RS:0;b543b351fe95:37213 {}] regionserver.HRegionServer(1482): Serving as b543b351fe95,37213,1733463591764, RpcServer on b543b351fe95/172.17.0.2:37213, sessionid=0x100ae0a5c73000e 2024-12-06T05:39:52,496 DEBUG [RS:0;b543b351fe95:37213 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T05:39:52,496 DEBUG [RS:0;b543b351fe95:37213 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b543b351fe95,37213,1733463591764 2024-12-06T05:39:52,496 DEBUG [RS:0;b543b351fe95:37213 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b543b351fe95,37213,1733463591764' 2024-12-06T05:39:52,496 DEBUG [RS:0;b543b351fe95:37213 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T05:39:52,498 DEBUG [RS:0;b543b351fe95:37213 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T05:39:52,498 DEBUG [RS:0;b543b351fe95:37213 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T05:39:52,498 DEBUG [RS:0;b543b351fe95:37213 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T05:39:52,498 DEBUG [RS:0;b543b351fe95:37213 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b543b351fe95,37213,1733463591764 2024-12-06T05:39:52,498 DEBUG [RS:0;b543b351fe95:37213 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b543b351fe95,37213,1733463591764' 2024-12-06T05:39:52,498 DEBUG [RS:0;b543b351fe95:37213 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T05:39:52,499 DEBUG [RS:0;b543b351fe95:37213 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T05:39:52,499 DEBUG [RS:0;b543b351fe95:37213 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T05:39:52,500 INFO [RS:0;b543b351fe95:37213 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T05:39:52,500 INFO [RS:0;b543b351fe95:37213 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T05:39:52,515 INFO [RS:1;b543b351fe95:37497 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T05:39:52,515 INFO [RS:1;b543b351fe95:37497 {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,37497,1733463591802-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:52,515 INFO [RS:1;b543b351fe95:37497 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:52,516 INFO [RS:1;b543b351fe95:37497 {}] regionserver.Replication(171): b543b351fe95,37497,1733463591802 started 2024-12-06T05:39:52,539 INFO [RS:1;b543b351fe95:37497 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:52,540 INFO [RS:1;b543b351fe95:37497 {}] regionserver.HRegionServer(1482): Serving as b543b351fe95,37497,1733463591802, RpcServer on b543b351fe95/172.17.0.2:37497, sessionid=0x100ae0a5c73000f 2024-12-06T05:39:52,540 DEBUG [RS:1;b543b351fe95:37497 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T05:39:52,540 DEBUG [RS:1;b543b351fe95:37497 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b543b351fe95,37497,1733463591802 2024-12-06T05:39:52,540 DEBUG [RS:1;b543b351fe95:37497 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b543b351fe95,37497,1733463591802' 2024-12-06T05:39:52,540 DEBUG [RS:1;b543b351fe95:37497 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T05:39:52,540 DEBUG [RS:1;b543b351fe95:37497 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T05:39:52,541 DEBUG [RS:1;b543b351fe95:37497 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T05:39:52,541 DEBUG [RS:1;b543b351fe95:37497 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T05:39:52,541 DEBUG [RS:1;b543b351fe95:37497 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b543b351fe95,37497,1733463591802 2024-12-06T05:39:52,541 DEBUG [RS:1;b543b351fe95:37497 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b543b351fe95,37497,1733463591802' 2024-12-06T05:39:52,541 DEBUG [RS:1;b543b351fe95:37497 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T05:39:52,542 DEBUG [RS:1;b543b351fe95:37497 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T05:39:52,542 DEBUG [RS:1;b543b351fe95:37497 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T05:39:52,542 INFO [RS:1;b543b351fe95:37497 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T05:39:52,542 INFO [RS:1;b543b351fe95:37497 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T05:39:52,600 INFO [RS:0;b543b351fe95:37213 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:39:52,603 INFO [RS:0;b543b351fe95:37213 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b543b351fe95%2C37213%2C1733463591764, suffix=, logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,37213,1733463591764, archiveDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/oldWALs, maxLogs=32 2024-12-06T05:39:52,624 DEBUG [RS:0;b543b351fe95:37213 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,37213,1733463591764/b543b351fe95%2C37213%2C1733463591764.1733463592604, exclude list is [], retry=0 2024-12-06T05:39:52,628 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-80ddb16c-ec66-4e39-a44c-0a80fa143251,DISK] 2024-12-06T05:39:52,628 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39561,DS-4485eaa6-367b-4084-97e1-b834fb8d8596,DISK] 2024-12-06T05:39:52,640 INFO [RS:0;b543b351fe95:37213 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,37213,1733463591764/b543b351fe95%2C37213%2C1733463591764.1733463592604 2024-12-06T05:39:52,640 DEBUG [RS:0;b543b351fe95:37213 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44855:44855),(127.0.0.1/127.0.0.1:43559:43559)] 2024-12-06T05:39:52,643 INFO [RS:1;b543b351fe95:37497 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:39:52,645 INFO [RS:1;b543b351fe95:37497 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b543b351fe95%2C37497%2C1733463591802, suffix=, logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,37497,1733463591802, archiveDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/oldWALs, maxLogs=32 2024-12-06T05:39:52,665 DEBUG [RS:1;b543b351fe95:37497 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,37497,1733463591802/b543b351fe95%2C37497%2C1733463591802.1733463592646, exclude list is [], retry=0 2024-12-06T05:39:52,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39561,DS-4485eaa6-367b-4084-97e1-b834fb8d8596,DISK] 2024-12-06T05:39:52,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-80ddb16c-ec66-4e39-a44c-0a80fa143251,DISK] 2024-12-06T05:39:52,682 INFO [RS:1;b543b351fe95:37497 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,37497,1733463591802/b543b351fe95%2C37497%2C1733463591802.1733463592646 2024-12-06T05:39:52,685 DEBUG [RS:1;b543b351fe95:37497 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43559:43559),(127.0.0.1/127.0.0.1:44855:44855)] 2024-12-06T05:39:52,692 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-06T05:39:52,692 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:39:52,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741868_1045 (size=32) 2024-12-06T05:39:52,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741868_1045 (size=32) 2024-12-06T05:39:52,711 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:52,713 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-06T05:39:52,715 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T05:39:52,715 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:52,716 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:52,716 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-06T05:39:52,717 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T05:39:52,717 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:52,718 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:52,719 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-06T05:39:52,721 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T05:39:52,721 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:52,722 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:52,722 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-06T05:39:52,724 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T05:39:52,724 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:52,725 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:52,725 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T05:39:52,726 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/hbase/meta/1588230740 2024-12-06T05:39:52,726 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/hbase/meta/1588230740 2024-12-06T05:39:52,728 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T05:39:52,728 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T05:39:52,729 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T05:39:52,731 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T05:39:52,739 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:39:52,740 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62826411, jitterRate=-0.06381352245807648}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T05:39:52,741 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733463592712Initializing all the Stores at 1733463592712Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463592713 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463592713Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463592713Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463592713Cleaning up temporary data from old regions at 1733463592728 (+15 ms)Region opened successfully at 1733463592741 (+13 ms) 2024-12-06T05:39:52,741 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T05:39:52,741 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T05:39:52,741 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T05:39:52,741 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T05:39:52,741 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T05:39:52,742 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T05:39:52,742 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733463592741Disabling compacts and flushes for region at 1733463592741Disabling writes for close at 1733463592741Writing region close event to WAL at 1733463592742 (+1 ms)Closed at 1733463592742 2024-12-06T05:39:52,744 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T05:39:52,744 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-06T05:39:52,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-06T05:39:52,746 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T05:39:52,748 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-06T05:39:52,898 DEBUG [b543b351fe95:39613 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-06T05:39:52,899 DEBUG [b543b351fe95:39613 {}] balancer.BalancerClusterState(204): Hosts are {b543b351fe95=0} racks are {/default-rack=0} 2024-12-06T05:39:52,902 DEBUG [b543b351fe95:39613 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-06T05:39:52,902 DEBUG [b543b351fe95:39613 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-06T05:39:52,902 DEBUG [b543b351fe95:39613 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-06T05:39:52,902 DEBUG [b543b351fe95:39613 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-06T05:39:52,902 INFO [b543b351fe95:39613 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-06T05:39:52,902 INFO [b543b351fe95:39613 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-06T05:39:52,902 DEBUG [b543b351fe95:39613 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-06T05:39:52,903 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b543b351fe95,37213,1733463591764 2024-12-06T05:39:52,905 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b543b351fe95,37213,1733463591764, state=OPENING 2024-12-06T05:39:52,906 DEBUG [PEWorker-4 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-06T05:39:52,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37213-0x100ae0a5c73000e, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:52,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:52,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37497-0x100ae0a5c73000f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:52,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39613-0x100ae0a5c73000c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:52,908 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:52,908 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:52,908 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:52,909 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:52,909 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T05:39:52,910 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b543b351fe95,37213,1733463591764}] 2024-12-06T05:39:53,066 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T05:39:53,068 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56973, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T05:39:53,072 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-06T05:39:53,073 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:39:53,073 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-06T05:39:53,075 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b543b351fe95%2C37213%2C1733463591764.meta, suffix=.meta, logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,37213,1733463591764, archiveDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/oldWALs, maxLogs=32 2024-12-06T05:39:53,094 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,37213,1733463591764/b543b351fe95%2C37213%2C1733463591764.meta.1733463593076.meta, exclude list is [], retry=0 2024-12-06T05:39:53,098 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39561,DS-4485eaa6-367b-4084-97e1-b834fb8d8596,DISK] 2024-12-06T05:39:53,098 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-80ddb16c-ec66-4e39-a44c-0a80fa143251,DISK] 2024-12-06T05:39:53,107 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,37213,1733463591764/b543b351fe95%2C37213%2C1733463591764.meta.1733463593076.meta 2024-12-06T05:39:53,109 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43559:43559),(127.0.0.1/127.0.0.1:44855:44855)] 2024-12-06T05:39:53,109 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:39:53,110 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-06T05:39:53,110 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-06T05:39:53,110 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-06T05:39:53,110 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-06T05:39:53,110 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:53,110 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-06T05:39:53,110 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-06T05:39:53,113 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-06T05:39:53,114 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T05:39:53,114 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:53,115 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:53,115 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-06T05:39:53,116 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T05:39:53,116 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:53,123 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:53,124 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-06T05:39:53,125 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T05:39:53,125 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:53,126 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:53,126 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-06T05:39:53,128 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T05:39:53,128 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:53,129 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:53,129 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T05:39:53,131 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/hbase/meta/1588230740 2024-12-06T05:39:53,132 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/hbase/meta/1588230740 2024-12-06T05:39:53,136 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T05:39:53,137 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T05:39:53,137 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T05:39:53,140 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T05:39:53,141 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73223110, jitterRate=0.09110936522483826}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T05:39:53,141 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-06T05:39:53,142 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733463593110Writing region info on filesystem at 1733463593110Initializing all the Stores at 1733463593112 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463593112Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463593112Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463593112Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463593112Cleaning up temporary data from old regions at 1733463593137 (+25 ms)Running coprocessor post-open hooks at 1733463593141 (+4 ms)Region opened successfully at 1733463593142 (+1 ms) 2024-12-06T05:39:53,144 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733463593065 2024-12-06T05:39:53,148 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-06T05:39:53,149 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-06T05:39:53,150 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b543b351fe95,37213,1733463591764 2024-12-06T05:39:53,152 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b543b351fe95,37213,1733463591764, state=OPEN 2024-12-06T05:39:53,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:39:53,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37497-0x100ae0a5c73000f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:39:53,154 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:53,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39613-0x100ae0a5c73000c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:39:53,154 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:53,154 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:53,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37213-0x100ae0a5c73000e, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:39:53,154 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:53,154 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b543b351fe95,37213,1733463591764 2024-12-06T05:39:53,160 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-06T05:39:53,161 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b543b351fe95,37213,1733463591764 in 245 msec 2024-12-06T05:39:53,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-06T05:39:53,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 417 msec 2024-12-06T05:39:53,168 DEBUG [PEWorker-3 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T05:39:53,168 INFO [PEWorker-3 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-06T05:39:53,172 DEBUG [PEWorker-3 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T05:39:53,172 DEBUG [PEWorker-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b543b351fe95,37213,1733463591764, seqNum=-1] 2024-12-06T05:39:53,172 DEBUG [PEWorker-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T05:39:53,190 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42909, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T05:39:53,211 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0240 sec 2024-12-06T05:39:53,212 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733463593212, completionTime=-1 2024-12-06T05:39:53,212 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-06T05:39:53,212 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-06T05:39:53,217 INFO [master/b543b351fe95:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-06T05:39:53,217 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733463653217 2024-12-06T05:39:53,217 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733463713217 2024-12-06T05:39:53,217 INFO [master/b543b351fe95:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec 2024-12-06T05:39:53,217 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-06T05:39:53,219 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,39613,1733463591658-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:53,219 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,39613,1733463591658-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:53,219 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,39613,1733463591658-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:53,219 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b543b351fe95:39613, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:53,219 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:53,221 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:53,232 DEBUG [master/b543b351fe95:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-06T05:39:53,244 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.330sec 2024-12-06T05:39:53,252 INFO [master/b543b351fe95:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-06T05:39:53,252 INFO [master/b543b351fe95:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-06T05:39:53,252 INFO [master/b543b351fe95:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-06T05:39:53,252 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-06T05:39:53,252 INFO [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-06T05:39:53,252 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,39613,1733463591658-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T05:39:53,253 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,39613,1733463591658-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-06T05:39:53,260 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-06T05:39:53,261 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-06T05:39:53,261 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,39613,1733463591658-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:53,289 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x179dbf20 to 127.0.0.1:58086 with session timeout=1000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5a485309 2024-12-06T05:39:53,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58ca535a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T05:39:53,297 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T05:39:53,301 DEBUG [ReadOnlyZKClient-127.0.0.1:58086@0x179dbf20 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b543b351fe95,37213,1733463591764, seqNum=-1] 2024-12-06T05:39:53,301 DEBUG [ReadOnlyZKClient-127.0.0.1:58086@0x179dbf20 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T05:39:53,304 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52568, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T05:39:53,307 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=b543b351fe95,39613,1733463591658 2024-12-06T05:39:53,307 INFO [Time-limited test {}] hbase.TestZooKeeper(138): Starting testMasterZKSessionRecoveryFailure 2024-12-06T05:39:53,307 ERROR [Time-limited test {}] master.HMaster(3218): Master server abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-06T05:39:53,307 ERROR [Time-limited test {}] master.HMaster(3223): ***** ABORTING master b543b351fe95,39613,1733463591658: Test recovery from zk session expired ***** org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired at org.apache.hadoop.hbase.TestZooKeeper.testMasterZKSessionRecoveryFailure(TestZooKeeper.java:141) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:39:53,307 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master 'b543b351fe95,39613,1733463591658' ***** 2024-12-06T05:39:53,307 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: Stopped by Time-limited test 2024-12-06T05:39:53,308 INFO [M:0;b543b351fe95:39613 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T05:39:53,308 INFO [M:0;b543b351fe95:39613 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;b543b351fe95:39613. 2024-12-06T05:39:53,308 DEBUG [M:0;b543b351fe95:39613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T05:39:53,308 DEBUG [M:0;b543b351fe95:39613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:39:53,308 INFO [M:0;b543b351fe95:39613 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T05:39:53,308 DEBUG [M:0;b543b351fe95:39613 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-06T05:39:53,308 DEBUG [M:0;b543b351fe95:39613 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-06T05:39:53,308 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-06T05:39:53,308 INFO [M:0;b543b351fe95:39613 {}] hbase.ChoreService(370): Chore service for: master/b543b351fe95:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-06T05:39:53,308 DEBUG [master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.large.0-1733463592224 {}] cleaner.HFileCleaner(306): Exit Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.large.0-1733463592224,5,FailOnTimeoutGroup] 2024-12-06T05:39:53,308 DEBUG [master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.small.0-1733463592225 {}] cleaner.HFileCleaner(306): Exit Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.small.0-1733463592225,5,FailOnTimeoutGroup] 2024-12-06T05:39:53,308 INFO [M:0;b543b351fe95:39613 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T05:39:53,311 DEBUG [M:0;b543b351fe95:39613 {}] master.HMaster(1795): Stopping service threads 2024-12-06T05:39:53,311 INFO [M:0;b543b351fe95:39613 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-06T05:39:53,311 INFO [M:0;b543b351fe95:39613 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-06T05:39:53,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37497-0x100ae0a5c73000f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:53,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37213-0x100ae0a5c73000e, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:53,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:53,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37497-0x100ae0a5c73000f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:53,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37213-0x100ae0a5c73000e, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:53,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:53,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39613-0x100ae0a5c73000c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:53,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39613-0x100ae0a5c73000c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:53,312 INFO [M:0;b543b351fe95:39613 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=true 2024-12-06T05:39:53,312 INFO [Time-limited test {}] hbase.TestZooKeeper(153): Creating table testMasterZKSessionRecoveryFailure_1733463593312 2024-12-06T05:39:53,312 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-06T05:39:53,312 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T05:39:53,313 DEBUG [M:0;b543b351fe95:39613 {}] zookeeper.ZKUtil(347): master:39613-0x100ae0a5c73000c, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-06T05:39:53,313 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37497-0x100ae0a5c73000f, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:39:53,313 WARN [M:0;b543b351fe95:39613 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-06T05:39:53,313 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:39:53,313 DEBUG [zk-event-processor-pool-0 {}] master.ActiveMasterManager(204): No master available. Notifying waiting threads 2024-12-06T05:39:53,314 INFO [M:0;b543b351fe95:39613 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/.lastflushedseqids 2024-12-06T05:39:53,314 DEBUG [ReadOnlyZKClient-127.0.0.1:58086@0x179dbf20 {}] client.ConnectionUtils(551): Failed to fetch master stub from registry org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase/master at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:39:53,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37497-0x100ae0a5c73000f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:53,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:53,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37497-0x100ae0a5c73000f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:53,316 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): regionserver:37213-0x100ae0a5c73000e, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T05:39:53,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39613-0x100ae0a5c73000c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:53,317 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T05:39:53,317 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T05:39:53,318 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b543b351fe95,40467,1733463591718 from backup master directory 2024-12-06T05:39:53,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b543b351fe95,40467,1733463591718 2024-12-06T05:39:53,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37497-0x100ae0a5c73000f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:53,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:53,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37213-0x100ae0a5c73000e, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:53,319 WARN [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T05:39:53,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39613-0x100ae0a5c73000c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:53,320 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b543b351fe95,40467,1733463591718 2024-12-06T05:39:53,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741870_1047 (size=99) 2024-12-06T05:39:53,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741870_1047 (size=99) 2024-12-06T05:39:53,331 INFO [M:0;b543b351fe95:39613 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-06T05:39:53,331 INFO [M:0;b543b351fe95:39613 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=true 2024-12-06T05:39:53,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741864_1041 (size=9664) 2024-12-06T05:39:53,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741864_1041 (size=9664) 2024-12-06T05:39:53,340 DEBUG [M:0;b543b351fe95:39613 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T05:39:53,340 INFO [M:0;b543b351fe95:39613 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:53,340 DEBUG [M:0;b543b351fe95:39613 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:53,340 DEBUG [M:0;b543b351fe95:39613 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T05:39:53,340 DEBUG [M:0;b543b351fe95:39613 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:53,341 ERROR [M:0;b543b351fe95:39613 {}] regionserver.HRegion(1960): Memstore data size is 7867 in region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:53,341 INFO [M:0;b543b351fe95:39613 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:53,341 DEBUG [M:0;b543b351fe95:39613 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733463593340Disabling compacts and flushes for region at 1733463593340Disabling writes for close at 1733463593340Writing region close event to WAL at 1733463593341 (+1 ms)Closed at 1733463593341 2024-12-06T05:39:53,341 INFO [M:0;b543b351fe95:39613 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-06T05:39:53,341 INFO [M:0;b543b351fe95:39613 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39613 2024-12-06T05:39:53,342 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T05:39:53,343 INFO [M:0;b543b351fe95:39613 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T05:39:53,346 INFO [master/b543b351fe95:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:53,346 INFO [master/b543b351fe95:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-06T05:39:53,355 INFO [master/b543b351fe95:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=1) cost 9ms. 2024-12-06T05:39:53,357 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-14-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server b543b351fe95:39613 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b543b351fe95/172.17.0.2:39613 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:39:53,358 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-14-2 {}] ipc.FailedServers(52): Added failed server with address b543b351fe95:39613 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b543b351fe95/172.17.0.2:39613 2024-12-06T05:39:53,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741871_1048 (size=196) 2024-12-06T05:39:53,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741871_1048 (size=196) 2024-12-06T05:39:53,374 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T05:39:53,374 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-06T05:39:53,375 INFO [master/b543b351fe95:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:39:53,383 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(342): old store file tracker DEFAULT is the same with new store file tracker, skip migration 2024-12-06T05:39:53,387 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(316): Renamed hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,39613,1733463591658 to hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,39613,1733463591658-dead as it is dead 2024-12-06T05:39:53,387 INFO [master/b543b351fe95:0:becomeActiveMaster {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,39613,1733463591658-dead/b543b351fe95%2C39613%2C1733463591658.1733463592051 2024-12-06T05:39:53,388 INFO [master/b543b351fe95:0:becomeActiveMaster {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,39613,1733463591658-dead/b543b351fe95%2C39613%2C1733463591658.1733463592051 after 0ms 2024-12-06T05:39:53,388 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(328): Renamed hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,39613,1733463591658-dead/b543b351fe95%2C39613%2C1733463591658.1733463592051 to hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.wals/b543b351fe95%2C39613%2C1733463591658.1733463592051 2024-12-06T05:39:53,388 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(330): Delete empty local region wal dir hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,39613,1733463591658-dead 2024-12-06T05:39:53,390 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,40467,1733463591718 2024-12-06T05:39:53,392 INFO [master/b543b351fe95:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:39:53,393 INFO [master/b543b351fe95:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b543b351fe95%2C40467%2C1733463591718, suffix=, logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,40467,1733463591718, archiveDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/oldWALs, maxLogs=10 2024-12-06T05:39:53,408 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-14-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server b543b351fe95:39613 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b543b351fe95/172.17.0.2:39613 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:39:53,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-14-1 {}] ipc.FailedServers(52): Added failed server with address b543b351fe95:39613 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b543b351fe95/172.17.0.2:39613 2024-12-06T05:39:53,410 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,40467,1733463591718/b543b351fe95%2C40467%2C1733463591718.1733463593394, exclude list is [], retry=0 2024-12-06T05:39:53,415 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39561,DS-4485eaa6-367b-4084-97e1-b834fb8d8596,DISK] 2024-12-06T05:39:53,415 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-80ddb16c-ec66-4e39-a44c-0a80fa143251,DISK] 2024-12-06T05:39:53,419 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T05:39:53,420 DEBUG [ReadOnlyZKClient-127.0.0.1:58086@0x179dbf20 {}] client.AsyncConnectionImpl(321): The fetched master address is b543b351fe95,40467,1733463591718 2024-12-06T05:39:53,420 DEBUG [ReadOnlyZKClient-127.0.0.1:58086@0x179dbf20 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5b665d58 2024-12-06T05:39:53,420 DEBUG [ReadOnlyZKClient-127.0.0.1:58086@0x179dbf20 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T05:39:53,424 INFO [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41154, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T05:39:53,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40467 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.checkInitialized(HMaster.java:3340) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.createTable(HMaster.java:2437) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.createTable(MasterRpcServices.java:786) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:53,429 INFO [master/b543b351fe95:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,40467,1733463591718/b543b351fe95%2C40467%2C1733463591718.1733463593394 2024-12-06T05:39:53,435 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43559:43559),(127.0.0.1/127.0.0.1:44855:44855)] 2024-12-06T05:39:53,435 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:39:53,435 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:53,435 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:53,435 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:53,441 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:53,443 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-06T05:39:53,443 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:53,444 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:53,444 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:53,445 INFO [M:0;b543b351fe95:39613 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T05:39:53,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39613-0x100ae0a5c73000c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:39:53,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39613-0x100ae0a5c73000c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:39:53,448 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-06T05:39:53,448 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:53,449 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:39:53,449 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:53,450 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-06T05:39:53,451 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:53,451 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:39:53,452 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:53,453 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-06T05:39:53,454 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:53,455 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:39:53,455 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:53,457 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(5516): Found 1 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.wals 2024-12-06T05:39:53,457 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.wals/b543b351fe95%2C39613%2C1733463591658.1733463592051 2024-12-06T05:39:53,459 DEBUG [RS:0;b543b351fe95:37213 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T05:39:53,461 INFO [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57545, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T05:39:53,462 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40467 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:53,467 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(5793): Applied 36, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=28, path=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.wals/b543b351fe95%2C39613%2C1733463591658.1733463592051 2024-12-06T05:39:53,467 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.68 KB heapSize=11.36 KB 2024-12-06T05:39:53,488 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2371a2ff144f439c9b8bba6cd39686fe is 82, key is hbase:meta,,1/info:regioninfo/1733463593150/Put/seqid=0 2024-12-06T05:39:53,510 DEBUG [RS:1;b543b351fe95:37497 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T05:39:53,511 INFO [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55695, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T05:39:53,512 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40467 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:53,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741873_1050 (size=5672) 2024-12-06T05:39:53,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741873_1050 (size=5672) 2024-12-06T05:39:53,518 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2371a2ff144f439c9b8bba6cd39686fe 2024-12-06T05:39:53,545 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6a3a07513dcf4a70bbe46c4165b0799b is 366, key is \x00\x00\x00\x00\x00\x00\x00\x02/proc:d/1733463593161/Put/seqid=0 2024-12-06T05:39:53,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741874_1051 (size=12229) 2024-12-06T05:39:53,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741874_1051 (size=12229) 2024-12-06T05:39:53,556 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6a3a07513dcf4a70bbe46c4165b0799b 2024-12-06T05:39:53,566 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40467 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:53,591 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/40a2eb2226354e498cd43ab00fc2a380 is 69, key is b543b351fe95,37213,1733463591764/rs:state/1733463592331/Put/seqid=0 2024-12-06T05:39:53,615 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40467 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:53,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40467 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.checkInitialized(HMaster.java:3340) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.createTable(HMaster.java:2437) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.createTable(MasterRpcServices.java:786) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:53,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741875_1052 (size=5224) 2024-12-06T05:39:53,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741875_1052 (size=5224) 2024-12-06T05:39:53,642 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/40a2eb2226354e498cd43ab00fc2a380 2024-12-06T05:39:53,653 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2371a2ff144f439c9b8bba6cd39686fe as hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2371a2ff144f439c9b8bba6cd39686fe 2024-12-06T05:39:53,665 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2371a2ff144f439c9b8bba6cd39686fe, entries=8, sequenceid=28, filesize=5.5 K 2024-12-06T05:39:53,667 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6a3a07513dcf4a70bbe46c4165b0799b as hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6a3a07513dcf4a70bbe46c4165b0799b 2024-12-06T05:39:53,669 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40467 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:53,679 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6a3a07513dcf4a70bbe46c4165b0799b, entries=26, sequenceid=28, filesize=11.9 K 2024-12-06T05:39:53,681 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/40a2eb2226354e498cd43ab00fc2a380 as hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/40a2eb2226354e498cd43ab00fc2a380 2024-12-06T05:39:53,691 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/40a2eb2226354e498cd43ab00fc2a380, entries=2, sequenceid=28, filesize=5.1 K 2024-12-06T05:39:53,692 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.68 KB/7867, heapSize ~11.06 KB/11328, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 225ms, sequenceid=28, compaction requested=false; wal=null 2024-12-06T05:39:53,693 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.wals/b543b351fe95%2C39613%2C1733463591658.1733463592051 2024-12-06T05:39:53,695 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:53,695 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:53,696 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T05:39:53,698 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:53,703 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/28.seqid, newMaxSeqId=28, maxSeqId=1 2024-12-06T05:39:53,705 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=29; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68236225, jitterRate=0.016798987984657288}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T05:39:53,705 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733463593436Initializing all the Stores at 1733463593437 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463593438 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463593439 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463593439Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463593439Obtaining lock to block concurrent updates at 1733463593467 (+28 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733463593467Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7867, getHeapSize=11568, getOffHeapSize=0, getCellsCount=36 at 1733463593467Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733463593467Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733463593468 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733463593488 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733463593488Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733463593528 (+40 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733463593544 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733463593544Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733463593568 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733463593590 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733463593590Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@fcc3b1a: reopening flushed file at 1733463593652 (+62 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8bc5f61: reopening flushed file at 1733463593666 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ccfdf11: reopening flushed file at 1733463593679 (+13 ms)Finished flush of dataSize ~7.68 KB/7867, heapSize ~11.06 KB/11328, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 225ms, sequenceid=28, compaction requested=false; wal=null at 1733463593692 (+13 ms)Cleaning up temporary data from old regions at 1733463593695 (+3 ms)Region opened successfully at 1733463593705 (+10 ms) 2024-12-06T05:39:53,706 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-06T05:39:53,712 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fd0e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b543b351fe95/172.17.0.2:0 2024-12-06T05:39:53,716 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(884): The info family in master local region already has data in it, skip migrating... 2024-12-06T05:39:53,717 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-06T05:39:53,717 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-06T05:39:53,717 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-06T05:39:53,717 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40467 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:53,719 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-06T05:39:53,726 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-06T05:39:53,727 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-12-06T05:39:53,727 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-06T05:39:53,736 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b543b351fe95,37213,1733463591764, regionLocation=b543b351fe95,37213,1733463591764, openSeqNum=2 2024-12-06T05:39:53,736 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b543b351fe95,37213,1733463591764, table=hbase:meta, region=1588230740 2024-12-06T05:39:53,736 INFO [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b543b351fe95,37213,1733463591764, state=OPEN 2024-12-06T05:39:53,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:39:53,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37497-0x100ae0a5c73000f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:39:53,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37213-0x100ae0a5c73000e, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:39:53,765 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:53,765 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:53,765 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:53,772 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 2 possibly 'live' servers, and 0 'splitting'. 2024-12-06T05:39:53,773 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40467 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:53,774 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ServerManager(517): Registering regionserver=b543b351fe95,37213,1733463591764 2024-12-06T05:39:53,775 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ServerManager(517): Registering regionserver=b543b351fe95,37497,1733463591802 2024-12-06T05:39:53,775 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-06T05:39:53,776 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-06T05:39:53,777 INFO [master/b543b351fe95:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-06T05:39:53,778 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-06T05:39:53,778 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-06T05:39:53,779 INFO [master/b543b351fe95:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-06T05:39:53,782 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-06T05:39:53,783 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-06T05:39:53,784 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-06T05:39:53,785 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-06T05:39:53,794 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-06T05:39:53,794 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-06T05:39:53,795 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=b543b351fe95,40467,1733463591718, sessionid=0x100ae0a5c73000d, setting cluster-up flag (Was=true) 2024-12-06T05:39:53,797 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-06T05:39:53,798 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b543b351fe95,40467,1733463591718 2024-12-06T05:39:53,801 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-06T05:39:53,802 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b543b351fe95,40467,1733463591718 2024-12-06T05:39:53,805 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ServerManager(1190): begin to load .lastflushedseqids at hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/.lastflushedseqids 2024-12-06T05:39:53,813 INFO [master/b543b351fe95:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-06T05:39:53,813 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b543b351fe95,40467,1733463591718 Number of backup masters: 0 Number of live region servers: 2 b543b351fe95,37213,1733463591764 b543b351fe95,37497,1733463591802 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-06T05:39:53,815 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:39:53,815 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:39:53,815 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:39:53,815 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:39:53,815 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b543b351fe95:0, corePoolSize=10, maxPoolSize=10 2024-12-06T05:39:53,815 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:53,815 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b543b351fe95:0, corePoolSize=2, maxPoolSize=2 2024-12-06T05:39:53,815 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:53,821 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40467 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:53,843 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733463623842 2024-12-06T05:39:53,843 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-06T05:39:53,843 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-06T05:39:53,843 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-06T05:39:53,843 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-06T05:39:53,843 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-06T05:39:53,843 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-06T05:39:53,844 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:53,845 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-06T05:39:53,846 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-06T05:39:53,846 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-06T05:39:53,854 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-06T05:39:53,854 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-06T05:39:53,854 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.large.0-1733463593854,5,FailOnTimeoutGroup] 2024-12-06T05:39:53,854 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.small.0-1733463593854,5,FailOnTimeoutGroup] 2024-12-06T05:39:53,854 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:53,854 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-06T05:39:53,855 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:53,855 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:53,855 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733463593855, completionTime=-1 2024-12-06T05:39:53,855 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-06T05:39:53,855 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-06T05:39:53,855 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T05:39:53,855 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b543b351fe95,37213,1733463591764, seqNum=-1] 2024-12-06T05:39:53,856 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T05:39:53,857 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44391, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T05:39:53,858 INFO [master/b543b351fe95:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-06T05:39:53,858 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733463653858 2024-12-06T05:39:53,858 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733463713858 2024-12-06T05:39:53,858 INFO [master/b543b351fe95:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-12-06T05:39:53,859 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,40467,1733463591718-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:53,859 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,40467,1733463591718-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:53,859 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,40467,1733463591718-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:53,859 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b543b351fe95:40467, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:53,859 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:53,860 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:53,862 DEBUG [master/b543b351fe95:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-06T05:39:53,874 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.554sec 2024-12-06T05:39:53,881 INFO [master/b543b351fe95:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-06T05:39:53,881 INFO [master/b543b351fe95:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-06T05:39:53,881 INFO [master/b543b351fe95:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-06T05:39:53,881 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-06T05:39:53,881 INFO [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-06T05:39:53,881 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,40467,1733463591718-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T05:39:53,881 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,40467,1733463591718-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-06T05:39:53,891 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-06T05:39:53,891 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-06T05:39:53,891 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,40467,1733463591718-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:53,940 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40467 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testMasterZKSessionRecoveryFailure_1733463593312', {NAME => 'fam', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T05:39:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40467 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testMasterZKSessionRecoveryFailure_1733463593312 2024-12-06T05:39:53,944 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testMasterZKSessionRecoveryFailure_1733463593312 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T05:39:53,945 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:53,945 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40467 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testMasterZKSessionRecoveryFailure_1733463593312" procId is: 4 2024-12-06T05:39:53,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40467 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T05:39:53,947 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testMasterZKSessionRecoveryFailure_1733463593312 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T05:39:53,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741876_1053 (size=399) 2024-12-06T05:39:53,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741876_1053 (size=399) 2024-12-06T05:39:54,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40467 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T05:39:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40467 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T05:39:54,379 INFO [RegionOpenAndInit-testMasterZKSessionRecoveryFailure_1733463593312-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ef38c2222a77faf047a5dc1902d4bddf, NAME => 'testMasterZKSessionRecoveryFailure_1733463593312,,1733463593940.ef38c2222a77faf047a5dc1902d4bddf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testMasterZKSessionRecoveryFailure_1733463593312', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'fam', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:39:54,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741877_1054 (size=83) 2024-12-06T05:39:54,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741877_1054 (size=83) 2024-12-06T05:39:54,401 DEBUG [RegionOpenAndInit-testMasterZKSessionRecoveryFailure_1733463593312-pool-0 {}] regionserver.HRegion(898): Instantiated testMasterZKSessionRecoveryFailure_1733463593312,,1733463593940.ef38c2222a77faf047a5dc1902d4bddf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:54,401 DEBUG [RegionOpenAndInit-testMasterZKSessionRecoveryFailure_1733463593312-pool-0 {}] regionserver.HRegion(1722): Closing ef38c2222a77faf047a5dc1902d4bddf, disabling compactions & flushes 2024-12-06T05:39:54,401 INFO [RegionOpenAndInit-testMasterZKSessionRecoveryFailure_1733463593312-pool-0 {}] regionserver.HRegion(1755): Closing region testMasterZKSessionRecoveryFailure_1733463593312,,1733463593940.ef38c2222a77faf047a5dc1902d4bddf. 2024-12-06T05:39:54,402 DEBUG [RegionOpenAndInit-testMasterZKSessionRecoveryFailure_1733463593312-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testMasterZKSessionRecoveryFailure_1733463593312,,1733463593940.ef38c2222a77faf047a5dc1902d4bddf. 2024-12-06T05:39:54,402 DEBUG [RegionOpenAndInit-testMasterZKSessionRecoveryFailure_1733463593312-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testMasterZKSessionRecoveryFailure_1733463593312,,1733463593940.ef38c2222a77faf047a5dc1902d4bddf. after waiting 0 ms 2024-12-06T05:39:54,402 DEBUG [RegionOpenAndInit-testMasterZKSessionRecoveryFailure_1733463593312-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testMasterZKSessionRecoveryFailure_1733463593312,,1733463593940.ef38c2222a77faf047a5dc1902d4bddf. 2024-12-06T05:39:54,402 INFO [RegionOpenAndInit-testMasterZKSessionRecoveryFailure_1733463593312-pool-0 {}] regionserver.HRegion(1973): Closed testMasterZKSessionRecoveryFailure_1733463593312,,1733463593940.ef38c2222a77faf047a5dc1902d4bddf. 2024-12-06T05:39:54,402 DEBUG [RegionOpenAndInit-testMasterZKSessionRecoveryFailure_1733463593312-pool-0 {}] regionserver.HRegion(1676): Region close journal for ef38c2222a77faf047a5dc1902d4bddf: Waiting for close lock at 1733463594401Disabling compacts and flushes for region at 1733463594401Disabling writes for close at 1733463594402 (+1 ms)Writing region close event to WAL at 1733463594402Closed at 1733463594402 2024-12-06T05:39:54,404 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testMasterZKSessionRecoveryFailure_1733463593312 execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T05:39:54,405 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testMasterZKSessionRecoveryFailure_1733463593312,,1733463593940.ef38c2222a77faf047a5dc1902d4bddf.","families":{"info":[{"qualifier":"regioninfo","vlen":82,"tag":[],"timestamp":"1733463594404"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733463594404"}]},"ts":"1733463594404"} 2024-12-06T05:39:54,407 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T05:39:54,409 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testMasterZKSessionRecoveryFailure_1733463593312 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T05:39:54,409 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testMasterZKSessionRecoveryFailure_1733463593312","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733463594409"}]},"ts":"1733463594409"} 2024-12-06T05:39:54,412 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testMasterZKSessionRecoveryFailure_1733463593312, state=ENABLING in hbase:meta 2024-12-06T05:39:54,413 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {b543b351fe95=0} racks are {/default-rack=0} 2024-12-06T05:39:54,414 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-06T05:39:54,414 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-06T05:39:54,414 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-06T05:39:54,414 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-06T05:39:54,414 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-06T05:39:54,415 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-06T05:39:54,415 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-06T05:39:54,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testMasterZKSessionRecoveryFailure_1733463593312, region=ef38c2222a77faf047a5dc1902d4bddf, ASSIGN}] 2024-12-06T05:39:54,417 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testMasterZKSessionRecoveryFailure_1733463593312, region=ef38c2222a77faf047a5dc1902d4bddf, ASSIGN 2024-12-06T05:39:54,419 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testMasterZKSessionRecoveryFailure_1733463593312, region=ef38c2222a77faf047a5dc1902d4bddf, ASSIGN; state=OFFLINE, location=b543b351fe95,37213,1733463591764; forceNewPlan=false, retain=false 2024-12-06T05:39:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40467 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T05:39:54,570 INFO [b543b351fe95:40467 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-06T05:39:54,570 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ef38c2222a77faf047a5dc1902d4bddf, regionState=OPENING, regionLocation=b543b351fe95,37213,1733463591764 2024-12-06T05:39:54,573 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-14-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testMasterZKSessionRecoveryFailure_1733463593312, region=ef38c2222a77faf047a5dc1902d4bddf, ASSIGN because future has completed 2024-12-06T05:39:54,573 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef38c2222a77faf047a5dc1902d4bddf, server=b543b351fe95,37213,1733463591764}] 2024-12-06T05:39:54,727 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T05:39:54,728 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60357, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T05:39:54,732 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testMasterZKSessionRecoveryFailure_1733463593312,,1733463593940.ef38c2222a77faf047a5dc1902d4bddf. 2024-12-06T05:39:54,732 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ef38c2222a77faf047a5dc1902d4bddf, NAME => 'testMasterZKSessionRecoveryFailure_1733463593312,,1733463593940.ef38c2222a77faf047a5dc1902d4bddf.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:39:54,733 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMasterZKSessionRecoveryFailure_1733463593312 ef38c2222a77faf047a5dc1902d4bddf 2024-12-06T05:39:54,733 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testMasterZKSessionRecoveryFailure_1733463593312,,1733463593940.ef38c2222a77faf047a5dc1902d4bddf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:54,733 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ef38c2222a77faf047a5dc1902d4bddf 2024-12-06T05:39:54,733 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ef38c2222a77faf047a5dc1902d4bddf 2024-12-06T05:39:54,735 INFO [StoreOpener-ef38c2222a77faf047a5dc1902d4bddf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family fam of region ef38c2222a77faf047a5dc1902d4bddf 2024-12-06T05:39:54,737 INFO [StoreOpener-ef38c2222a77faf047a5dc1902d4bddf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef38c2222a77faf047a5dc1902d4bddf columnFamilyName fam 2024-12-06T05:39:54,737 DEBUG [StoreOpener-ef38c2222a77faf047a5dc1902d4bddf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:54,737 INFO [StoreOpener-ef38c2222a77faf047a5dc1902d4bddf-1 {}] regionserver.HStore(327): Store=ef38c2222a77faf047a5dc1902d4bddf/fam, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:39:54,737 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ef38c2222a77faf047a5dc1902d4bddf 2024-12-06T05:39:54,738 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testMasterZKSessionRecoveryFailure_1733463593312/ef38c2222a77faf047a5dc1902d4bddf 2024-12-06T05:39:54,739 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testMasterZKSessionRecoveryFailure_1733463593312/ef38c2222a77faf047a5dc1902d4bddf 2024-12-06T05:39:54,739 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ef38c2222a77faf047a5dc1902d4bddf 2024-12-06T05:39:54,739 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ef38c2222a77faf047a5dc1902d4bddf 2024-12-06T05:39:54,741 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ef38c2222a77faf047a5dc1902d4bddf 2024-12-06T05:39:54,743 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testMasterZKSessionRecoveryFailure_1733463593312/ef38c2222a77faf047a5dc1902d4bddf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:39:54,744 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ef38c2222a77faf047a5dc1902d4bddf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73419309, jitterRate=0.09403295814990997}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:39:54,744 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ef38c2222a77faf047a5dc1902d4bddf 2024-12-06T05:39:54,744 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ef38c2222a77faf047a5dc1902d4bddf: Running coprocessor pre-open hook at 1733463594734Writing region info on filesystem at 1733463594734Initializing all the Stores at 1733463594735 (+1 ms)Instantiating store for column family {NAME => 'fam', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463594735Cleaning up temporary data from old regions at 1733463594739 (+4 ms)Running coprocessor post-open hooks at 1733463594744 (+5 ms)Region opened successfully at 1733463594744 2024-12-06T05:39:54,746 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testMasterZKSessionRecoveryFailure_1733463593312,,1733463593940.ef38c2222a77faf047a5dc1902d4bddf., pid=6, masterSystemTime=1733463594726 2024-12-06T05:39:54,748 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testMasterZKSessionRecoveryFailure_1733463593312,,1733463593940.ef38c2222a77faf047a5dc1902d4bddf. 2024-12-06T05:39:54,749 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testMasterZKSessionRecoveryFailure_1733463593312,,1733463593940.ef38c2222a77faf047a5dc1902d4bddf. 2024-12-06T05:39:54,749 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ef38c2222a77faf047a5dc1902d4bddf, regionState=OPEN, openSeqNum=2, regionLocation=b543b351fe95,37213,1733463591764 2024-12-06T05:39:54,752 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-14-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef38c2222a77faf047a5dc1902d4bddf, server=b543b351fe95,37213,1733463591764 because future has completed 2024-12-06T05:39:54,763 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-06T05:39:54,763 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ef38c2222a77faf047a5dc1902d4bddf, server=b543b351fe95,37213,1733463591764 in 182 msec 2024-12-06T05:39:54,768 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-06T05:39:54,769 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testMasterZKSessionRecoveryFailure_1733463593312, region=ef38c2222a77faf047a5dc1902d4bddf, ASSIGN in 348 msec 2024-12-06T05:39:54,772 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testMasterZKSessionRecoveryFailure_1733463593312 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T05:39:54,772 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testMasterZKSessionRecoveryFailure_1733463593312","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733463594772"}]},"ts":"1733463594772"} 2024-12-06T05:39:54,775 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testMasterZKSessionRecoveryFailure_1733463593312, state=ENABLED in hbase:meta 2024-12-06T05:39:54,777 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testMasterZKSessionRecoveryFailure_1733463593312 execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T05:39:54,781 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testMasterZKSessionRecoveryFailure_1733463593312 in 837 msec 2024-12-06T05:39:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40467 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T05:39:55,081 INFO [RPCClient-NioEventLoopGroup-6-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testMasterZKSessionRecoveryFailure_1733463593312 completed 2024-12-06T05:39:55,081 INFO [Time-limited test {}] hbase.TestZooKeeper(164): Putting table testMasterZKSessionRecoveryFailure_1733463593312 2024-12-06T05:39:55,085 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testMasterZKSessionRecoveryFailure_1733463593312', row='testrow', locateType=CURRENT is [region=testMasterZKSessionRecoveryFailure_1733463593312,,1733463593940.ef38c2222a77faf047a5dc1902d4bddf., hostname=b543b351fe95,37213,1733463591764, seqNum=2] 2024-12-06T05:39:55,090 DEBUG [Time-limited test {}] master.HMaster(3213): Abort called but aborted=true, stopped=true 2024-12-06T05:39:55,090 ERROR [Time-limited test {}] master.HMaster(3218): Master server abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-06T05:39:55,091 ERROR [Time-limited test {}] master.HMaster(3225): ***** ABORTING master b543b351fe95,40467,1733463591718: killAll ***** 2024-12-06T05:39:55,091 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master 'b543b351fe95,40467,1733463591718' ***** 2024-12-06T05:39:55,091 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: Stopped by Time-limited test 2024-12-06T05:39:55,091 INFO [M:1;b543b351fe95:40467 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T05:39:55,091 INFO [M:1;b543b351fe95:40467 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:1;b543b351fe95:40467. 2024-12-06T05:39:55,091 DEBUG [M:1;b543b351fe95:40467 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T05:39:55,091 DEBUG [M:1;b543b351fe95:40467 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:39:55,091 INFO [M:1;b543b351fe95:40467 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T05:39:55,091 DEBUG [M:1;b543b351fe95:40467 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-06T05:39:55,091 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-06T05:39:55,091 DEBUG [M:1;b543b351fe95:40467 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-06T05:39:55,091 DEBUG [master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.large.0-1733463593854 {}] cleaner.HFileCleaner(306): Exit Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.large.0-1733463593854,5,FailOnTimeoutGroup] 2024-12-06T05:39:55,091 DEBUG [master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.small.0-1733463593854 {}] cleaner.HFileCleaner(306): Exit Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.small.0-1733463593854,5,FailOnTimeoutGroup] 2024-12-06T05:39:55,091 INFO [M:1;b543b351fe95:40467 {}] hbase.ChoreService(370): Chore service for: master/b543b351fe95:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-06T05:39:55,091 INFO [M:1;b543b351fe95:40467 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T05:39:55,092 DEBUG [M:1;b543b351fe95:40467 {}] master.HMaster(1795): Stopping service threads 2024-12-06T05:39:55,092 INFO [M:1;b543b351fe95:40467 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-06T05:39:55,092 INFO [M:1;b543b351fe95:40467 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-06T05:39:55,093 INFO [M:1;b543b351fe95:40467 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=true 2024-12-06T05:39:55,093 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-06T05:39:55,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:55,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37213-0x100ae0a5c73000e, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:55,095 DEBUG [M:1;b543b351fe95:40467 {}] zookeeper.ZKUtil(347): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-06T05:39:55,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37497-0x100ae0a5c73000f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:55,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:55,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37213-0x100ae0a5c73000e, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:55,095 WARN [M:1;b543b351fe95:40467 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-06T05:39:55,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37497-0x100ae0a5c73000f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:55,095 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server b543b351fe95,37213,1733463591764: killAll ***** 2024-12-06T05:39:55,095 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-06T05:39:55,095 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-06T05:39:55,096 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-06T05:39:55,096 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37213-0x100ae0a5c73000e, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:39:55,097 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-06T05:39:55,097 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-06T05:39:55,097 INFO [M:1;b543b351fe95:40467 {}] master.ServerManager(1134): Rewriting .lastflushedseqids file at: hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/.lastflushedseqids 2024-12-06T05:39:55,099 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 564727976 }, "NonHeapMemoryUsage": { "committed": 161021952, "init": 7667712, "max": -1, "used": 157074152 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 7149, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 29, "ProcessCallTime_min": 0, "ProcessCallTime_max": 11, "ProcessCallTime_mean": 1, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 1, "ProcessCallTime_75th_percentile": 2, "ProcessCallTime_90th_percentile": 2, "ProcessCallTime_95th_percentile": 11, "ProcessCallTime_98th_percentile": 11, "ProcessCallTime_99th_percentile": 11, "ProcessCallTime_99.9th_percentile": 11, "ProcessCallTime_TimeRangeCount_0-1": 9, "ProcessCallTime_TimeRangeCount_1-3": 2, "ProcessCallTime_TimeRangeCount_10-30": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 29, "QueueCallTime_min": 0, "QueueCallTime_max": 0, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 12, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 29, "TotalCallTime_min": 0, "TotalCallTime_max": 11, "TotalCallTime_mean": 1, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 1, "TotalCallTime_75th_percentile": 2, "TotalCallTime_90th_percentile": 2, "TotalCallTime_95th_percentile": 11, "TotalCallTime_98th_percentile": 11, "TotalCallTime_99th_percentile": 11, "TotalCallTime_99.9th_percentile": 11, "TotalCallTime_TimeRangeCount_0-1": 9, "TotalCallTime_TimeRangeCount_1-3": 2, "TotalCallTime_TimeRangeCount_10-30": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 29, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 51, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 18, "ResponseSize_75th_percentile": 22, "ResponseSize_90th_percentile": 172, "ResponseSize_95th_percentile": 173, "ResponseSize_98th_percentile": 173, "ResponseSize_99th_percentile": 173, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 5, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 3, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 29, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 106, "RequestSize_25th_percentile": 92, "RequestSize_median": 104, "RequestSize_75th_percentile": 119, "RequestSize_90th_percentile": 120, "RequestSize_95th_percentile": 120, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_10-100": 5, "RequestSize_SizeRangeCount_100-1000": 7, "sentBytes": 1935 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:58086", "tag.serverName": "b543b351fe95,37775,1733463581162", "tag.clusterId": "f24feb01-2567-4d5e-a4c6-86f800bb290a", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95", "regionCount": 1, "storeCount": 4, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 74, "memStoreHeapSize": 1248, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733463581162, "averageRegionSize": 74, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 15057, "localBytesRead": 15057, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.4, "writeRequestRatePerSecond": 0.4, "ByteBuffAllocatorHeapAllocationBytes": 15185, "ByteBuffAllocatorPoolAllocationBytes": 332800, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 11, "totalRowActionRequestCount": 9, "readRequestCount": 7, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 2, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 7, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 2, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 7, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 7, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 2, "Put_min": 2, "Put_max": 2, "Put_mean": 2, "Put_25th_percentile": 2, "Put_median": 2, "Put_75th_percentile": 2, "Put_90th_percentile": 2, "Put_95th_percentile": 2, "Put_98th_percentile": 2, "Put_99th_percentile": 2, "Put_99.9th_percentile": 2, "Put_TimeRangeCount_1-3": 2, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 0, "Get_max": 1, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 1, "Get_75th_percentile": 1, "Get_90th_percentile": 1, "Get_95th_percentile": 1, "Get_98th_percentile": 1, "Get_99th_percentile": 1, "Get_99.9th_percentile": 1, "Get_TimeRangeCount_0-1": 2, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 7, "ScanSize_min": 0, "ScanSize_max": 144, "ScanSize_mean": 61, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 144, "ScanSize_90th_percentile": 144, "ScanSize_95th_percentile": 144, "ScanSize_98th_percentile": 144, "ScanSize_99th_percentile": 144, "ScanSize_99.9th_percentile": 144, "ScanSize_SizeRangeCount_0-10": 4, "ScanSize_SizeRangeCount_100-1000": 3, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-06T05:39:55,100 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37497-0x100ae0a5c73000f, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:39:55,102 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b543b351fe95,40467,1733463591718 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:39:55,102 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b543b351fe95,40467,1733463591718 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:39:55,103 WARN [Time-limited test {}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.regionserver.RegionServerAbortedException): Server b543b351fe95,40467,1733463591718 aborting at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:?] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:?] at org.apache.hadoop.hbase.Abortable.abort(Abortable.java:45) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster.killAll(SingleProcessHBaseCluster.java:880) ~[test-classes/:?] at org.apache.hadoop.hbase.TestZooKeeper.after(TestZooKeeper.java:107) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: Server b543b351fe95,40467,1733463591718 aborting at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-12-06T05:39:55,104 DEBUG [RS:0;b543b351fe95:37213 {}] regionserver.HRegionServer(2589): No master found; retry 2024-12-06T05:39:55,105 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b543b351fe95,37213,1733463591764' ***** 2024-12-06T05:39:55,105 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: killAll 2024-12-06T05:39:55,105 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server b543b351fe95,37497,1733463591802: killAll ***** 2024-12-06T05:39:55,105 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-06T05:39:55,106 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-06T05:39:55,106 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-06T05:39:55,106 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-06T05:39:55,107 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-06T05:39:55,108 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 566825128 }, "NonHeapMemoryUsage": { "committed": 161087488, "init": 7667712, "max": -1, "used": 157110376 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 7149, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 29, "ProcessCallTime_min": 0, "ProcessCallTime_max": 11, "ProcessCallTime_mean": 1, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 1, "ProcessCallTime_75th_percentile": 2, "ProcessCallTime_90th_percentile": 2, "ProcessCallTime_95th_percentile": 11, "ProcessCallTime_98th_percentile": 11, "ProcessCallTime_99th_percentile": 11, "ProcessCallTime_99.9th_percentile": 11, "ProcessCallTime_TimeRangeCount_0-1": 9, "ProcessCallTime_TimeRangeCount_1-3": 2, "ProcessCallTime_TimeRangeCount_10-30": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 29, "QueueCallTime_min": 0, "QueueCallTime_max": 0, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 12, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 29, "TotalCallTime_min": 0, "TotalCallTime_max": 11, "TotalCallTime_mean": 1, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 1, "TotalCallTime_75th_percentile": 2, "TotalCallTime_90th_percentile": 2, "TotalCallTime_95th_percentile": 11, "TotalCallTime_98th_percentile": 11, "TotalCallTime_99th_percentile": 11, "TotalCallTime_99.9th_percentile": 11, "TotalCallTime_TimeRangeCount_0-1": 9, "TotalCallTime_TimeRangeCount_1-3": 2, "TotalCallTime_TimeRangeCount_10-30": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 29, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 51, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 18, "ResponseSize_75th_percentile": 22, "ResponseSize_90th_percentile": 172, "ResponseSize_95th_percentile": 173, "ResponseSize_98th_percentile": 173, "ResponseSize_99th_percentile": 173, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 5, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 3, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 29, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 106, "RequestSize_25th_percentile": 92, "RequestSize_median": 104, "RequestSize_75th_percentile": 119, "RequestSize_90th_percentile": 120, "RequestSize_95th_percentile": 120, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_10-100": 5, "RequestSize_SizeRangeCount_100-1000": 7, "sentBytes": 1935 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:58086", "tag.serverName": "b543b351fe95,37775,1733463581162", "tag.clusterId": "f24feb01-2567-4d5e-a4c6-86f800bb290a", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95", "regionCount": 1, "storeCount": 4, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 74, "memStoreHeapSize": 1248, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733463581162, "averageRegionSize": 74, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 15057, "localBytesRead": 15057, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.4, "writeRequestRatePerSecond": 0.4, "ByteBuffAllocatorHeapAllocationBytes": 15185, "ByteBuffAllocatorPoolAllocationBytes": 332800, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 11, "totalRowActionRequestCount": 9, "readRequestCount": 7, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 2, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 7, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 2, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 7, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 7, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 2, "Put_min": 2, "Put_max": 2, "Put_mean": 2, "Put_25th_percentile": 2, "Put_median": 2, "Put_75th_percentile": 2, "Put_90th_percentile": 2, "Put_95th_percentile": 2, "Put_98th_percentile": 2, "Put_99th_percentile": 2, "Put_99.9th_percentile": 2, "Put_TimeRangeCount_1-3": 2, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 0, "Get_max": 1, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 1, "Get_75th_percentile": 1, "Get_90th_percentile": 1, "Get_95th_percentile": 1, "Get_98th_percentile": 1, "Get_99th_percentile": 1, "Get_99.9th_percentile": 1, "Get_TimeRangeCount_0-1": 2, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 7, "ScanSize_min": 0, "ScanSize_max": 144, "ScanSize_mean": 61, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 144, "ScanSize_90th_percentile": 144, "ScanSize_95th_percentile": 144, "ScanSize_98th_percentile": 144, "ScanSize_99th_percentile": 144, "ScanSize_99.9th_percentile": 144, "ScanSize_SizeRangeCount_0-10": 4, "ScanSize_SizeRangeCount_100-1000": 3, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-06T05:39:55,110 DEBUG [HMaster-EventLoopGroup-12-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b543b351fe95,40467,1733463591718 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:39:55,111 WARN [Time-limited test {}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.regionserver.RegionServerAbortedException): Server b543b351fe95,40467,1733463591718 aborting at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:?] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:?] at org.apache.hadoop.hbase.Abortable.abort(Abortable.java:45) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster.killAll(SingleProcessHBaseCluster.java:880) ~[test-classes/:?] at org.apache.hadoop.hbase.TestZooKeeper.after(TestZooKeeper.java:107) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: Server b543b351fe95,40467,1733463591718 aborting at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-12-06T05:39:55,113 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b543b351fe95,37497,1733463591802' ***** 2024-12-06T05:39:55,113 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: killAll 2024-12-06T05:39:55,113 INFO [RS:1;b543b351fe95:37497 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T05:39:55,113 INFO [RS:1;b543b351fe95:37497 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-06T05:39:55,113 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T05:39:55,113 INFO [RS:1;b543b351fe95:37497 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-06T05:39:55,114 INFO [RS:1;b543b351fe95:37497 {}] regionserver.HRegionServer(956): aborting server b543b351fe95,37497,1733463591802 2024-12-06T05:39:55,114 INFO [RS:1;b543b351fe95:37497 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T05:39:55,114 INFO [RS:1;b543b351fe95:37497 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;b543b351fe95:37497. 2024-12-06T05:39:55,114 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T05:39:55,114 DEBUG [RS:1;b543b351fe95:37497 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T05:39:55,114 DEBUG [RS:1;b543b351fe95:37497 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:39:55,114 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.TestZooKeeper.after(TestZooKeeper.java:110) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T05:39:55,114 INFO [RS:1;b543b351fe95:37497 {}] regionserver.HRegionServer(976): stopping server b543b351fe95,37497,1733463591802; all regions closed. 2024-12-06T05:39:55,114 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:39:55,115 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x179dbf20 to 127.0.0.1:58086 2024-12-06T05:39:55,116 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-06T05:39:55,122 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T05:39:55,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741878_1055 (size=147) 2024-12-06T05:39:55,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741878_1055 (size=147) 2024-12-06T05:39:55,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741867_1044 (size=93) 2024-12-06T05:39:55,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741867_1044 (size=93) 2024-12-06T05:39:55,137 INFO [M:1;b543b351fe95:40467 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-06T05:39:55,137 INFO [M:1;b543b351fe95:40467 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=true 2024-12-06T05:39:55,137 DEBUG [RS:1;b543b351fe95:37497 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:39:55,138 INFO [RS:1;b543b351fe95:37497 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T05:39:55,138 INFO [RS:1;b543b351fe95:37497 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T05:39:55,138 INFO [RS:1;b543b351fe95:37497 {}] hbase.ChoreService(370): Chore service for: regionserver/b543b351fe95:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-06T05:39:55,138 INFO [RS:1;b543b351fe95:37497 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T05:39:55,138 INFO [RS:1;b543b351fe95:37497 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T05:39:55,138 INFO [RS:1;b543b351fe95:37497 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T05:39:55,138 INFO [RS:1;b543b351fe95:37497 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T05:39:55,138 INFO [RS:1;b543b351fe95:37497 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37497 2024-12-06T05:39:55,138 INFO [regionserver/b543b351fe95:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T05:39:55,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T05:39:55,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37497-0x100ae0a5c73000f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b543b351fe95,37497,1733463591802 2024-12-06T05:39:55,140 INFO [RS:1;b543b351fe95:37497 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T05:39:55,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741872_1049 (size=17918) 2024-12-06T05:39:55,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741872_1049 (size=17918) 2024-12-06T05:39:55,145 DEBUG [M:1;b543b351fe95:40467 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T05:39:55,145 INFO [M:1;b543b351fe95:40467 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:55,145 DEBUG [M:1;b543b351fe95:40467 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:55,145 DEBUG [M:1;b543b351fe95:40467 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T05:39:55,145 DEBUG [M:1;b543b351fe95:40467 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:55,147 ERROR [M:1;b543b351fe95:40467 {}] regionserver.HRegion(1960): Memstore data size is 16081 in region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:55,147 INFO [M:1;b543b351fe95:40467 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:55,147 DEBUG [M:1;b543b351fe95:40467 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733463595145Disabling compacts and flushes for region at 1733463595145Disabling writes for close at 1733463595145Writing region close event to WAL at 1733463595147 (+2 ms)Closed at 1733463595147 2024-12-06T05:39:55,147 INFO [M:1;b543b351fe95:40467 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-06T05:39:55,147 INFO [M:1;b543b351fe95:40467 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40467 2024-12-06T05:39:55,150 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T05:39:55,150 INFO [M:1;b543b351fe95:40467 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T05:39:55,187 INFO [regionserver/b543b351fe95:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T05:39:55,241 INFO [RS:1;b543b351fe95:37497 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T05:39:55,241 INFO [RS:1;b543b351fe95:37497 {}] regionserver.HRegionServer(1031): Exiting; stopping=b543b351fe95,37497,1733463591802; zookeeper connection closed. 2024-12-06T05:39:55,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37497-0x100ae0a5c73000f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:39:55,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37497-0x100ae0a5c73000f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:39:55,242 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6e2a5adf {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6e2a5adf 2024-12-06T05:39:55,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:39:55,251 INFO [M:1;b543b351fe95:40467 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T05:39:55,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40467-0x100ae0a5c73000d, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:39:55,304 INFO [RS:0;b543b351fe95:37213 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T05:39:55,305 INFO [RS:0;b543b351fe95:37213 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-06T05:39:55,305 INFO [RS:0;b543b351fe95:37213 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-06T05:39:55,305 INFO [RS:0;b543b351fe95:37213 {}] regionserver.HRegionServer(3091): Received CLOSE for ef38c2222a77faf047a5dc1902d4bddf 2024-12-06T05:39:55,305 INFO [RS:0;b543b351fe95:37213 {}] regionserver.HRegionServer(956): aborting server b543b351fe95,37213,1733463591764 2024-12-06T05:39:55,305 INFO [RS:0;b543b351fe95:37213 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T05:39:55,305 INFO [RS:0;b543b351fe95:37213 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;b543b351fe95:37213. 2024-12-06T05:39:55,305 DEBUG [RS:0;b543b351fe95:37213 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T05:39:55,305 DEBUG [RS:0;b543b351fe95:37213 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:39:55,305 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ef38c2222a77faf047a5dc1902d4bddf, disabling compactions & flushes 2024-12-06T05:39:55,305 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testMasterZKSessionRecoveryFailure_1733463593312,,1733463593940.ef38c2222a77faf047a5dc1902d4bddf. 2024-12-06T05:39:55,306 INFO [RS:0;b543b351fe95:37213 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T05:39:55,306 INFO [RS:0;b543b351fe95:37213 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T05:39:55,306 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testMasterZKSessionRecoveryFailure_1733463593312,,1733463593940.ef38c2222a77faf047a5dc1902d4bddf. 2024-12-06T05:39:55,306 INFO [RS:0;b543b351fe95:37213 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T05:39:55,306 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testMasterZKSessionRecoveryFailure_1733463593312,,1733463593940.ef38c2222a77faf047a5dc1902d4bddf. after waiting 0 ms 2024-12-06T05:39:55,306 INFO [RS:0;b543b351fe95:37213 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-06T05:39:55,306 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testMasterZKSessionRecoveryFailure_1733463593312,,1733463593940.ef38c2222a77faf047a5dc1902d4bddf. 2024-12-06T05:39:55,310 INFO [RS:0;b543b351fe95:37213 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-06T05:39:55,310 DEBUG [RS:0;b543b351fe95:37213 {}] regionserver.HRegionServer(1325): Online Regions={ef38c2222a77faf047a5dc1902d4bddf=testMasterZKSessionRecoveryFailure_1733463593312,,1733463593940.ef38c2222a77faf047a5dc1902d4bddf., 1588230740=hbase:meta,,1.1588230740} 2024-12-06T05:39:55,310 DEBUG [RS:0;b543b351fe95:37213 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, ef38c2222a77faf047a5dc1902d4bddf 2024-12-06T05:39:55,310 ERROR [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1960): Memstore data size is 41 in region testMasterZKSessionRecoveryFailure_1733463593312,,1733463593940.ef38c2222a77faf047a5dc1902d4bddf. 2024-12-06T05:39:55,310 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testMasterZKSessionRecoveryFailure_1733463593312,,1733463593940.ef38c2222a77faf047a5dc1902d4bddf. 2024-12-06T05:39:55,311 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T05:39:55,311 INFO [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T05:39:55,311 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T05:39:55,311 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T05:39:55,311 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T05:39:55,311 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ef38c2222a77faf047a5dc1902d4bddf: Waiting for close lock at 1733463595305Running coprocessor pre-close hooks at 1733463595305Disabling compacts and flushes for region at 1733463595305Disabling writes for close at 1733463595306 (+1 ms)Writing region close event to WAL at 1733463595310 (+4 ms)Running coprocessor post-close hooks at 1733463595310Closed at 1733463595310 2024-12-06T05:39:55,311 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testMasterZKSessionRecoveryFailure_1733463593312,,1733463593940.ef38c2222a77faf047a5dc1902d4bddf. 2024-12-06T05:39:55,311 ERROR [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 1857 in region hbase:meta,,1.1588230740 2024-12-06T05:39:55,312 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-06T05:39:55,312 INFO [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T05:39:55,312 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733463595310Running coprocessor pre-close hooks at 1733463595310Disabling compacts and flushes for region at 1733463595310Disabling writes for close at 1733463595311 (+1 ms)Writing region close event to WAL at 1733463595311Running coprocessor post-close hooks at 1733463595312 (+1 ms)Closed at 1733463595312 2024-12-06T05:39:55,312 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-06T05:39:55,351 INFO [regionserver/b543b351fe95:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T05:39:55,445 INFO [regionserver/b543b351fe95:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-06T05:39:55,445 INFO [regionserver/b543b351fe95:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-06T05:39:55,510 INFO [RS:0;b543b351fe95:37213 {}] regionserver.HRegionServer(976): stopping server b543b351fe95,37213,1733463591764; all regions closed. 2024-12-06T05:39:55,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741869_1046 (size=2499) 2024-12-06T05:39:55,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741869_1046 (size=2499) 2024-12-06T05:39:55,515 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,37213,1733463591764/b543b351fe95%2C37213%2C1733463591764.meta.1733463593076.meta not finished, retry = 0 2024-12-06T05:39:55,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741866_1043 (size=623) 2024-12-06T05:39:55,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741866_1043 (size=623) 2024-12-06T05:39:55,621 DEBUG [RS:0;b543b351fe95:37213 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:39:55,621 INFO [RS:0;b543b351fe95:37213 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T05:39:55,621 INFO [RS:0;b543b351fe95:37213 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T05:39:55,622 INFO [RS:0;b543b351fe95:37213 {}] hbase.ChoreService(370): Chore service for: regionserver/b543b351fe95:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-06T05:39:55,622 INFO [RS:0;b543b351fe95:37213 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T05:39:55,622 INFO [regionserver/b543b351fe95:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T05:39:55,622 INFO [RS:0;b543b351fe95:37213 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37213 2024-12-06T05:39:55,624 INFO [RS:0;b543b351fe95:37213 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T05:39:55,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37213-0x100ae0a5c73000e, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b543b351fe95,37213,1733463591764 2024-12-06T05:39:55,624 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$360/0x00007fef248de4b8@49593a9b rejected from java.util.concurrent.ThreadPoolExecutor@c5badd4[Shutting down, pool size = 1, active threads = 0, queued tasks = 0, completed tasks = 19] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1360) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-12-06T05:39:55,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37213-0x100ae0a5c73000e, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:39:55,726 INFO [RS:0;b543b351fe95:37213 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T05:39:55,726 INFO [RS:0;b543b351fe95:37213 {}] regionserver.HRegionServer(1031): Exiting; stopping=b543b351fe95,37213,1733463591764; zookeeper connection closed. 2024-12-06T05:39:55,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37213-0x100ae0a5c73000e, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:39:55,726 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1a79bfe7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1a79bfe7 2024-12-06T05:39:55,726 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 2 master(s) and 2 regionserver(s) complete 2024-12-06T05:39:55,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73000b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:39:55,829 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:55,832 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=testing utility connecting to ZooKeeper ensemble=127.0.0.1:58086 2024-12-06T05:39:55,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility0x0, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:39:55,834 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): testing utility-0x100ae0a5c730011 connected 2024-12-06T05:39:55,835 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-06T05:39:55,835 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/meta-region-server 2024-12-06T05:39:55,835 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/rs 2024-12-06T05:39:55,836 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/splitWAL 2024-12-06T05:39:55,836 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters 2024-12-06T05:39:55,837 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc 2024-12-06T05:39:55,843 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/draining 2024-12-06T05:39:55,844 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/master-maintenance 2024-12-06T05:39:55,844 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot 2024-12-06T05:39:55,844 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/hbaseid 2024-12-06T05:39:55,845 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/table 2024-12-06T05:39:55,848 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc/abort 2024-12-06T05:39:55,848 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc/acquired 2024-12-06T05:39:55,849 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc/reached 2024-12-06T05:39:55,850 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot/abort 2024-12-06T05:39:55,850 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot/acquired 2024-12-06T05:39:55,851 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot/reached 2024-12-06T05:39:55,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/reached 2024-12-06T05:39:55,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/online-snapshot 2024-12-06T05:39:55,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/acquired 2024-12-06T05:39:55,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/abort 2024-12-06T05:39:55,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/reached 2024-12-06T05:39:55,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/flush-table-proc 2024-12-06T05:39:55,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/acquired 2024-12-06T05:39:55,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/abort 2024-12-06T05:39:55,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/table 2024-12-06T05:39:55,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:55,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/hbaseid 2024-12-06T05:39:55,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot 2024-12-06T05:39:55,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master-maintenance 2024-12-06T05:39:55,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/draining 2024-12-06T05:39:55,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc 2024-12-06T05:39:55,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:55,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/splitWAL 2024-12-06T05:39:55,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs 2024-12-06T05:39:55,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:39:55,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T05:39:55,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase 2024-12-06T05:39:55,870 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestZooKeeper#testMasterZKSessionRecoveryFailure Thread=345 (was 319) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-11-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2086754199_22 at /127.0.0.1:47418 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-11-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-14-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (268780456) connection to localhost/127.0.0.1:34453 from jenkins.hfs.5 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:34453 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-12-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (268780456) connection to localhost/127.0.0.1:34453 from jenkins.hfs.4 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: HMaster-EventLoopGroup-11-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-14-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:34453 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1471901137_22 at /127.0.0.1:58202 [Waiting for operation #35] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-14-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-12-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-12-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=632 (was 592) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=456 (was 408) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4553 (was 4642) 2024-12-06T05:39:55,880 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestZooKeeper#testLogSplittingAfterMasterRecoveryDueToZKExpiry Thread=345, OpenFileDescriptor=632, MaxFileDescriptor=1048576, SystemLoadAverage=456, ProcessCount=11, AvailableMemoryMB=4552 2024-12-06T05:39:55,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741879_1056 (size=7) 2024-12-06T05:39:55,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741879_1056 (size=7) 2024-12-06T05:39:55,905 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 with version=8 2024-12-06T05:39:55,905 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/hbase-staging 2024-12-06T05:39:55,907 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b543b351fe95:0 server-side Connection retries=45 2024-12-06T05:39:55,907 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:55,907 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:55,907 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T05:39:55,907 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:55,907 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T05:39:55,907 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-06T05:39:55,908 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T05:39:55,908 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42019 2024-12-06T05:39:55,909 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42019 connecting to ZooKeeper ensemble=127.0.0.1:58086 2024-12-06T05:39:55,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:420190x0, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:39:55,921 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42019-0x100ae0a5c730012 connected 2024-12-06T05:39:55,924 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:55,926 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:55,929 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T05:39:55,929 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279, hbase.cluster.distributed=false 2024-12-06T05:39:55,931 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T05:39:55,933 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42019 2024-12-06T05:39:55,937 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42019 2024-12-06T05:39:55,941 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42019 2024-12-06T05:39:55,942 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42019 2024-12-06T05:39:55,942 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42019 2024-12-06T05:39:55,944 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b543b351fe95:0 server-side Connection retries=45 2024-12-06T05:39:55,944 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:55,944 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:55,944 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T05:39:55,944 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:55,944 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T05:39:55,944 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-06T05:39:55,944 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T05:39:55,945 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42551 2024-12-06T05:39:55,946 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42551 connecting to ZooKeeper ensemble=127.0.0.1:58086 2024-12-06T05:39:55,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:425510x0, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:39:55,952 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:55,953 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42551-0x100ae0a5c730013 connected 2024-12-06T05:39:55,953 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:55,997 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T05:39:55,999 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T05:39:55,999 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279, hbase.cluster.distributed=false 2024-12-06T05:39:56,001 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T05:39:56,039 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42551 2024-12-06T05:39:56,057 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42551 2024-12-06T05:39:56,067 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42551 2024-12-06T05:39:56,072 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42551 2024-12-06T05:39:56,086 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42551 2024-12-06T05:39:56,109 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b543b351fe95:0 server-side Connection retries=45 2024-12-06T05:39:56,110 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:56,110 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:56,110 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T05:39:56,110 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:56,110 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T05:39:56,110 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T05:39:56,110 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T05:39:56,114 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42121 2024-12-06T05:39:56,115 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42121 connecting to ZooKeeper ensemble=127.0.0.1:58086 2024-12-06T05:39:56,116 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:56,119 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:56,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:421210x0, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:39:56,137 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:421210x0, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T05:39:56,138 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42121-0x100ae0a5c730014 connected 2024-12-06T05:39:56,138 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T05:39:56,142 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T05:39:56,143 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42121-0x100ae0a5c730014, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:39:56,146 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42121-0x100ae0a5c730014, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T05:39:56,147 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42121 2024-12-06T05:39:56,147 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42121 2024-12-06T05:39:56,153 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42121 2024-12-06T05:39:56,161 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42121 2024-12-06T05:39:56,163 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42121 2024-12-06T05:39:56,185 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b543b351fe95:0 server-side Connection retries=45 2024-12-06T05:39:56,185 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:56,185 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:56,185 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T05:39:56,185 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:56,185 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T05:39:56,185 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T05:39:56,186 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T05:39:56,188 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43157 2024-12-06T05:39:56,190 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43157 connecting to ZooKeeper ensemble=127.0.0.1:58086 2024-12-06T05:39:56,191 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:56,193 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:56,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:431570x0, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:39:56,226 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:431570x0, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T05:39:56,226 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T05:39:56,227 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43157-0x100ae0a5c730015 connected 2024-12-06T05:39:56,229 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T05:39:56,238 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43157-0x100ae0a5c730015, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:39:56,239 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43157-0x100ae0a5c730015, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T05:39:56,243 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43157 2024-12-06T05:39:56,244 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43157 2024-12-06T05:39:56,247 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43157 2024-12-06T05:39:56,261 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43157 2024-12-06T05:39:56,269 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43157 2024-12-06T05:39:56,289 DEBUG [M:0;b543b351fe95:42019 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b543b351fe95:42019 2024-12-06T05:39:56,290 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b543b351fe95,42019,1733463595907 2024-12-06T05:39:56,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43157-0x100ae0a5c730015, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:56,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:56,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42121-0x100ae0a5c730014, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:56,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:56,295 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b543b351fe95,42019,1733463595907 2024-12-06T05:39:56,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43157-0x100ae0a5c730015, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:56,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43157-0x100ae0a5c730015, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:56,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:56,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42121-0x100ae0a5c730014, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:56,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42121-0x100ae0a5c730014, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:56,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:56,304 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T05:39:56,305 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b543b351fe95,42019,1733463595907 from backup master directory 2024-12-06T05:39:56,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b543b351fe95,42019,1733463595907 2024-12-06T05:39:56,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:56,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43157-0x100ae0a5c730015, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:56,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:56,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42121-0x100ae0a5c730014, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:56,307 WARN [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T05:39:56,307 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b543b351fe95,42019,1733463595907 2024-12-06T05:39:56,309 DEBUG [M:1;b543b351fe95:42551 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:1;b543b351fe95:42551 2024-12-06T05:39:56,310 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b543b351fe95,42551,1733463595943 2024-12-06T05:39:56,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:56,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42121-0x100ae0a5c730014, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:56,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43157-0x100ae0a5c730015, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:56,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:56,314 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b543b351fe95,42551,1733463595943 2024-12-06T05:39:56,316 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/hbase.id] with ID: b7c25f36-05b2-43f3-92e5-304406a0c4c1 2024-12-06T05:39:56,316 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/.tmp/hbase.id 2024-12-06T05:39:56,317 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T05:39:56,317 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ActiveMasterManager(296): Another master is the active master, b543b351fe95,42019,1733463595907; waiting to become the next active master 2024-12-06T05:39:56,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741880_1057 (size=42) 2024-12-06T05:39:56,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741880_1057 (size=42) 2024-12-06T05:39:56,363 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/.tmp/hbase.id]:[hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/hbase.id] 2024-12-06T05:39:56,386 INFO [master/b543b351fe95:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:56,386 INFO [master/b543b351fe95:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-06T05:39:56,389 INFO [master/b543b351fe95:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms. 2024-12-06T05:39:56,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:56,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43157-0x100ae0a5c730015, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:56,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:56,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42121-0x100ae0a5c730014, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:56,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741881_1058 (size=196) 2024-12-06T05:39:56,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741881_1058 (size=196) 2024-12-06T05:39:56,442 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T05:39:56,443 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-06T05:39:56,453 INFO [master/b543b351fe95:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:39:56,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741882_1059 (size=1189) 2024-12-06T05:39:56,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741882_1059 (size=1189) 2024-12-06T05:39:56,508 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store 2024-12-06T05:39:56,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741883_1060 (size=34) 2024-12-06T05:39:56,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741883_1060 (size=34) 2024-12-06T05:39:56,550 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:56,550 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T05:39:56,551 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:56,551 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:56,551 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T05:39:56,551 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:56,551 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:56,551 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733463596550Disabling compacts and flushes for region at 1733463596550Disabling writes for close at 1733463596551 (+1 ms)Writing region close event to WAL at 1733463596551Closed at 1733463596551 2024-12-06T05:39:56,552 WARN [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/.initializing 2024-12-06T05:39:56,553 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,42019,1733463595907 2024-12-06T05:39:56,554 INFO [master/b543b351fe95:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:39:56,560 INFO [master/b543b351fe95:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b543b351fe95%2C42019%2C1733463595907, suffix=, logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,42019,1733463595907, archiveDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/oldWALs, maxLogs=10 2024-12-06T05:39:56,594 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,42019,1733463595907/b543b351fe95%2C42019%2C1733463595907.1733463596560, exclude list is [], retry=0 2024-12-06T05:39:56,601 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-80ddb16c-ec66-4e39-a44c-0a80fa143251,DISK] 2024-12-06T05:39:56,603 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39561,DS-4485eaa6-367b-4084-97e1-b834fb8d8596,DISK] 2024-12-06T05:39:56,610 INFO [master/b543b351fe95:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,42019,1733463595907/b543b351fe95%2C42019%2C1733463595907.1733463596560 2024-12-06T05:39:56,625 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44855:44855),(127.0.0.1/127.0.0.1:43559:43559)] 2024-12-06T05:39:56,625 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:39:56,626 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:56,626 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:56,626 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:56,636 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:56,639 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-06T05:39:56,640 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:56,641 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:56,641 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:56,643 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-06T05:39:56,643 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:56,644 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:39:56,645 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:56,655 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-06T05:39:56,655 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:56,656 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:39:56,656 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:56,659 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-06T05:39:56,659 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:56,661 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:39:56,661 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:56,662 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:56,663 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:56,665 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:56,665 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:56,666 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T05:39:56,669 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:56,674 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:39:56,675 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60717396, jitterRate=-0.09524029493331909}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T05:39:56,677 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733463596626Initializing all the Stores at 1733463596627 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463596628 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463596635 (+7 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463596635Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463596635Cleaning up temporary data from old regions at 1733463596665 (+30 ms)Region opened successfully at 1733463596677 (+12 ms) 2024-12-06T05:39:56,677 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-06T05:39:56,691 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29d1dc9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b543b351fe95/172.17.0.2:0 2024-12-06T05:39:56,692 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-06T05:39:56,692 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-06T05:39:56,692 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-06T05:39:56,693 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-06T05:39:56,694 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-06T05:39:56,695 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-06T05:39:56,695 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-06T05:39:56,699 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-06T05:39:56,701 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-06T05:39:56,702 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-06T05:39:56,703 INFO [master/b543b351fe95:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-06T05:39:56,704 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-06T05:39:56,705 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-06T05:39:56,705 INFO [master/b543b351fe95:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-06T05:39:56,710 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-06T05:39:56,711 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-06T05:39:56,712 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-06T05:39:56,713 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-06T05:39:56,716 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-06T05:39:56,716 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-06T05:39:56,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43157-0x100ae0a5c730015, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T05:39:56,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T05:39:56,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T05:39:56,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43157-0x100ae0a5c730015, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:56,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:56,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:56,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42121-0x100ae0a5c730014, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T05:39:56,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42121-0x100ae0a5c730014, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:56,722 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=b543b351fe95,42019,1733463595907, sessionid=0x100ae0a5c730012, setting cluster-up flag (Was=false) 2024-12-06T05:39:56,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:56,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43157-0x100ae0a5c730015, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:56,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42121-0x100ae0a5c730014, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:56,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:56,732 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-06T05:39:56,733 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b543b351fe95,42019,1733463595907 2024-12-06T05:39:56,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:56,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43157-0x100ae0a5c730015, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:56,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:56,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42121-0x100ae0a5c730014, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:56,759 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-06T05:39:56,760 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b543b351fe95,42019,1733463595907 2024-12-06T05:39:56,782 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-06T05:39:56,798 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-06T05:39:56,798 INFO [master/b543b351fe95:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-06T05:39:56,798 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b543b351fe95,42019,1733463595907 Number of backup masters: 1 b543b351fe95,42551,1733463595943 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-06T05:39:56,800 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:39:56,800 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:39:56,803 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:39:56,803 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:39:56,803 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b543b351fe95:0, corePoolSize=10, maxPoolSize=10 2024-12-06T05:39:56,803 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:56,803 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b543b351fe95:0, corePoolSize=2, maxPoolSize=2 2024-12-06T05:39:56,803 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:56,816 INFO [RS:0;b543b351fe95:42121 {}] regionserver.HRegionServer(746): ClusterId : b7c25f36-05b2-43f3-92e5-304406a0c4c1 2024-12-06T05:39:56,816 DEBUG [RS:0;b543b351fe95:42121 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T05:39:56,819 DEBUG [RS:0;b543b351fe95:42121 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T05:39:56,819 DEBUG [RS:0;b543b351fe95:42121 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T05:39:56,821 DEBUG [RS:0;b543b351fe95:42121 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T05:39:56,822 DEBUG [RS:0;b543b351fe95:42121 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2435364e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b543b351fe95/172.17.0.2:0 2024-12-06T05:39:56,839 INFO [RS:1;b543b351fe95:43157 {}] regionserver.HRegionServer(746): ClusterId : b7c25f36-05b2-43f3-92e5-304406a0c4c1 2024-12-06T05:39:56,839 DEBUG [RS:1;b543b351fe95:43157 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T05:39:56,852 DEBUG [RS:1;b543b351fe95:43157 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T05:39:56,852 DEBUG [RS:1;b543b351fe95:43157 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T05:39:56,854 DEBUG [RS:1;b543b351fe95:43157 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T05:39:56,854 DEBUG [RS:1;b543b351fe95:43157 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4687bf7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b543b351fe95/172.17.0.2:0 2024-12-06T05:39:56,854 DEBUG [RS:0;b543b351fe95:42121 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b543b351fe95:42121 2024-12-06T05:39:56,855 INFO [RS:0;b543b351fe95:42121 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T05:39:56,855 INFO [RS:0;b543b351fe95:42121 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T05:39:56,855 DEBUG [RS:0;b543b351fe95:42121 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T05:39:56,868 INFO [RS:0;b543b351fe95:42121 {}] regionserver.HRegionServer(2659): reportForDuty to master=b543b351fe95,42019,1733463595907 with port=42121, startcode=1733463596109 2024-12-06T05:39:56,870 DEBUG [RS:0;b543b351fe95:42121 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T05:39:56,876 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T05:39:56,876 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-06T05:39:56,877 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:56,877 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-06T05:39:56,905 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733463626905 2024-12-06T05:39:56,905 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-06T05:39:56,905 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-06T05:39:56,906 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-06T05:39:56,906 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-06T05:39:56,906 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-06T05:39:56,906 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-06T05:39:56,906 INFO [HMaster-EventLoopGroup-15-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57263, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T05:39:56,907 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42019 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:56,907 DEBUG [RS:0;b543b351fe95:42121 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-06T05:39:56,907 WARN [RS:0;b543b351fe95:42121 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-06T05:39:56,913 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:56,915 DEBUG [RS:1;b543b351fe95:43157 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;b543b351fe95:43157 2024-12-06T05:39:56,915 INFO [RS:1;b543b351fe95:43157 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T05:39:56,915 INFO [RS:1;b543b351fe95:43157 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T05:39:56,915 DEBUG [RS:1;b543b351fe95:43157 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T05:39:56,916 INFO [RS:1;b543b351fe95:43157 {}] regionserver.HRegionServer(2659): reportForDuty to master=b543b351fe95,42019,1733463595907 with port=43157, startcode=1733463596184 2024-12-06T05:39:56,916 DEBUG [RS:1;b543b351fe95:43157 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T05:39:56,921 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-06T05:39:56,921 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-06T05:39:56,921 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-06T05:39:56,924 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-06T05:39:56,924 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-06T05:39:56,924 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.large.0-1733463596924,5,FailOnTimeoutGroup] 2024-12-06T05:39:56,925 INFO [HMaster-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36497, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T05:39:56,926 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42019 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:56,928 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.small.0-1733463596924,5,FailOnTimeoutGroup] 2024-12-06T05:39:56,928 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:56,929 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-06T05:39:56,929 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:56,929 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:56,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741885_1062 (size=1321) 2024-12-06T05:39:56,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741885_1062 (size=1321) 2024-12-06T05:39:56,931 DEBUG [RS:1;b543b351fe95:43157 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-06T05:39:56,932 WARN [RS:1;b543b351fe95:43157 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-06T05:39:56,934 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-06T05:39:56,935 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:39:56,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741886_1063 (size=32) 2024-12-06T05:39:56,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741886_1063 (size=32) 2024-12-06T05:39:56,948 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:56,950 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-06T05:39:56,951 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T05:39:56,951 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:56,952 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:56,953 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-06T05:39:56,954 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T05:39:56,955 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:56,955 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:56,955 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-06T05:39:56,957 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T05:39:56,957 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:56,957 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:56,957 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-06T05:39:56,959 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T05:39:56,959 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:56,959 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:56,959 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T05:39:56,960 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/hbase/meta/1588230740 2024-12-06T05:39:56,960 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/hbase/meta/1588230740 2024-12-06T05:39:56,962 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T05:39:56,962 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T05:39:56,964 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T05:39:56,965 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T05:39:56,968 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:39:56,968 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71016659, jitterRate=0.0582306832075119}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T05:39:56,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733463596948Initializing all the Stores at 1733463596949 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463596949Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463596949Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463596949Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463596949Cleaning up temporary data from old regions at 1733463596962 (+13 ms)Region opened successfully at 1733463596969 (+7 ms) 2024-12-06T05:39:56,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T05:39:56,969 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T05:39:56,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T05:39:56,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T05:39:56,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T05:39:56,973 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T05:39:56,973 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733463596969Disabling compacts and flushes for region at 1733463596969Disabling writes for close at 1733463596969Writing region close event to WAL at 1733463596973 (+4 ms)Closed at 1733463596973 2024-12-06T05:39:56,975 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T05:39:56,975 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-06T05:39:56,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-06T05:39:56,976 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T05:39:56,978 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-06T05:39:57,008 INFO [RS:0;b543b351fe95:42121 {}] regionserver.HRegionServer(2659): reportForDuty to master=b543b351fe95,42019,1733463595907 with port=42121, startcode=1733463596109 2024-12-06T05:39:57,009 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42019 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b543b351fe95,42121,1733463596109 2024-12-06T05:39:57,010 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42019 {}] master.ServerManager(517): Registering regionserver=b543b351fe95,42121,1733463596109 2024-12-06T05:39:57,012 DEBUG [RS:0;b543b351fe95:42121 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:39:57,012 DEBUG [RS:0;b543b351fe95:42121 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34453 2024-12-06T05:39:57,012 DEBUG [RS:0;b543b351fe95:42121 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T05:39:57,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T05:39:57,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T05:39:57,014 DEBUG [RS:0;b543b351fe95:42121 {}] zookeeper.ZKUtil(111): regionserver:42121-0x100ae0a5c730014, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b543b351fe95,42121,1733463596109 2024-12-06T05:39:57,014 WARN [RS:0;b543b351fe95:42121 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T05:39:57,014 INFO [RS:0;b543b351fe95:42121 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:39:57,014 DEBUG [RS:0;b543b351fe95:42121 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,42121,1733463596109 2024-12-06T05:39:57,015 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b543b351fe95,42121,1733463596109] 2024-12-06T05:39:57,018 INFO [RS:0;b543b351fe95:42121 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T05:39:57,024 INFO [RS:0;b543b351fe95:42121 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T05:39:57,028 INFO [RS:0;b543b351fe95:42121 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T05:39:57,028 INFO [RS:0;b543b351fe95:42121 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,029 INFO [RS:0;b543b351fe95:42121 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T05:39:57,030 INFO [RS:0;b543b351fe95:42121 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T05:39:57,030 INFO [RS:0;b543b351fe95:42121 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,030 DEBUG [RS:0;b543b351fe95:42121 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,030 DEBUG [RS:0;b543b351fe95:42121 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,030 DEBUG [RS:0;b543b351fe95:42121 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,031 DEBUG [RS:0;b543b351fe95:42121 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,031 DEBUG [RS:0;b543b351fe95:42121 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,031 DEBUG [RS:0;b543b351fe95:42121 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0, corePoolSize=2, maxPoolSize=2 2024-12-06T05:39:57,031 DEBUG [RS:0;b543b351fe95:42121 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,031 DEBUG [RS:0;b543b351fe95:42121 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,031 DEBUG [RS:0;b543b351fe95:42121 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,031 DEBUG [RS:0;b543b351fe95:42121 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,031 DEBUG [RS:0;b543b351fe95:42121 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,031 DEBUG [RS:0;b543b351fe95:42121 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,031 DEBUG [RS:0;b543b351fe95:42121 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b543b351fe95:0, corePoolSize=3, maxPoolSize=3 2024-12-06T05:39:57,031 DEBUG [RS:0;b543b351fe95:42121 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b543b351fe95:0, corePoolSize=3, maxPoolSize=3 2024-12-06T05:39:57,033 INFO [RS:1;b543b351fe95:43157 {}] regionserver.HRegionServer(2659): reportForDuty to master=b543b351fe95,42019,1733463595907 with port=43157, startcode=1733463596184 2024-12-06T05:39:57,034 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42019 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b543b351fe95,43157,1733463596184 2024-12-06T05:39:57,034 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42019 {}] master.ServerManager(517): Registering regionserver=b543b351fe95,43157,1733463596184 2024-12-06T05:39:57,036 DEBUG [RS:1;b543b351fe95:43157 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:39:57,036 DEBUG [RS:1;b543b351fe95:43157 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34453 2024-12-06T05:39:57,036 DEBUG [RS:1;b543b351fe95:43157 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T05:39:57,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T05:39:57,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T05:39:57,038 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b543b351fe95,43157,1733463596184] 2024-12-06T05:39:57,038 DEBUG [RS:1;b543b351fe95:43157 {}] zookeeper.ZKUtil(111): regionserver:43157-0x100ae0a5c730015, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b543b351fe95,43157,1733463596184 2024-12-06T05:39:57,038 WARN [RS:1;b543b351fe95:43157 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T05:39:57,038 INFO [RS:1;b543b351fe95:43157 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:39:57,039 DEBUG [RS:1;b543b351fe95:43157 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,43157,1733463596184 2024-12-06T05:39:57,053 INFO [RS:0;b543b351fe95:42121 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,054 INFO [RS:0;b543b351fe95:42121 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,054 INFO [RS:0;b543b351fe95:42121 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,054 INFO [RS:0;b543b351fe95:42121 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,054 INFO [RS:0;b543b351fe95:42121 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,054 INFO [RS:0;b543b351fe95:42121 {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,42121,1733463596109-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T05:39:57,063 INFO [RS:1;b543b351fe95:43157 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T05:39:57,065 INFO [RS:1;b543b351fe95:43157 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T05:39:57,065 INFO [RS:1;b543b351fe95:43157 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T05:39:57,066 INFO [RS:1;b543b351fe95:43157 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,066 INFO [RS:1;b543b351fe95:43157 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T05:39:57,068 INFO [RS:1;b543b351fe95:43157 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T05:39:57,068 INFO [RS:1;b543b351fe95:43157 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,068 DEBUG [RS:1;b543b351fe95:43157 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,068 DEBUG [RS:1;b543b351fe95:43157 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,068 DEBUG [RS:1;b543b351fe95:43157 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,068 DEBUG [RS:1;b543b351fe95:43157 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,068 DEBUG [RS:1;b543b351fe95:43157 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,068 DEBUG [RS:1;b543b351fe95:43157 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0, corePoolSize=2, maxPoolSize=2 2024-12-06T05:39:57,068 DEBUG [RS:1;b543b351fe95:43157 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,068 DEBUG [RS:1;b543b351fe95:43157 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,069 DEBUG [RS:1;b543b351fe95:43157 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,069 DEBUG [RS:1;b543b351fe95:43157 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,069 DEBUG [RS:1;b543b351fe95:43157 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,069 DEBUG [RS:1;b543b351fe95:43157 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,069 DEBUG [RS:1;b543b351fe95:43157 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b543b351fe95:0, corePoolSize=3, maxPoolSize=3 2024-12-06T05:39:57,069 DEBUG [RS:1;b543b351fe95:43157 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b543b351fe95:0, corePoolSize=3, maxPoolSize=3 2024-12-06T05:39:57,079 INFO [RS:0;b543b351fe95:42121 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T05:39:57,080 INFO [RS:0;b543b351fe95:42121 {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,42121,1733463596109-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,080 INFO [RS:0;b543b351fe95:42121 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,080 INFO [RS:0;b543b351fe95:42121 {}] regionserver.Replication(171): b543b351fe95,42121,1733463596109 started 2024-12-06T05:39:57,087 INFO [RS:1;b543b351fe95:43157 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,087 INFO [RS:1;b543b351fe95:43157 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,087 INFO [RS:1;b543b351fe95:43157 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,087 INFO [RS:1;b543b351fe95:43157 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,087 INFO [RS:1;b543b351fe95:43157 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,087 INFO [RS:1;b543b351fe95:43157 {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,43157,1733463596184-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T05:39:57,098 INFO [RS:0;b543b351fe95:42121 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,098 INFO [RS:0;b543b351fe95:42121 {}] regionserver.HRegionServer(1482): Serving as b543b351fe95,42121,1733463596109, RpcServer on b543b351fe95/172.17.0.2:42121, sessionid=0x100ae0a5c730014 2024-12-06T05:39:57,098 DEBUG [RS:0;b543b351fe95:42121 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T05:39:57,098 DEBUG [RS:0;b543b351fe95:42121 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b543b351fe95,42121,1733463596109 2024-12-06T05:39:57,098 DEBUG [RS:0;b543b351fe95:42121 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b543b351fe95,42121,1733463596109' 2024-12-06T05:39:57,098 DEBUG [RS:0;b543b351fe95:42121 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T05:39:57,099 DEBUG [RS:0;b543b351fe95:42121 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T05:39:57,099 DEBUG [RS:0;b543b351fe95:42121 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T05:39:57,099 DEBUG [RS:0;b543b351fe95:42121 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T05:39:57,099 DEBUG [RS:0;b543b351fe95:42121 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b543b351fe95,42121,1733463596109 2024-12-06T05:39:57,099 DEBUG [RS:0;b543b351fe95:42121 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b543b351fe95,42121,1733463596109' 2024-12-06T05:39:57,099 DEBUG [RS:0;b543b351fe95:42121 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T05:39:57,099 DEBUG [RS:0;b543b351fe95:42121 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T05:39:57,099 DEBUG [RS:0;b543b351fe95:42121 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T05:39:57,100 INFO [RS:0;b543b351fe95:42121 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T05:39:57,100 INFO [RS:0;b543b351fe95:42121 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T05:39:57,110 INFO [RS:1;b543b351fe95:43157 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T05:39:57,110 INFO [RS:1;b543b351fe95:43157 {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,43157,1733463596184-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,111 INFO [RS:1;b543b351fe95:43157 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,111 INFO [RS:1;b543b351fe95:43157 {}] regionserver.Replication(171): b543b351fe95,43157,1733463596184 started 2024-12-06T05:39:57,128 WARN [b543b351fe95:42019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T05:39:57,130 INFO [RS:1;b543b351fe95:43157 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,131 INFO [RS:1;b543b351fe95:43157 {}] regionserver.HRegionServer(1482): Serving as b543b351fe95,43157,1733463596184, RpcServer on b543b351fe95/172.17.0.2:43157, sessionid=0x100ae0a5c730015 2024-12-06T05:39:57,131 DEBUG [RS:1;b543b351fe95:43157 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T05:39:57,131 DEBUG [RS:1;b543b351fe95:43157 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b543b351fe95,43157,1733463596184 2024-12-06T05:39:57,131 DEBUG [RS:1;b543b351fe95:43157 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b543b351fe95,43157,1733463596184' 2024-12-06T05:39:57,131 DEBUG [RS:1;b543b351fe95:43157 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T05:39:57,131 DEBUG [RS:1;b543b351fe95:43157 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T05:39:57,132 DEBUG [RS:1;b543b351fe95:43157 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T05:39:57,132 DEBUG [RS:1;b543b351fe95:43157 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T05:39:57,132 DEBUG [RS:1;b543b351fe95:43157 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b543b351fe95,43157,1733463596184 2024-12-06T05:39:57,132 DEBUG [RS:1;b543b351fe95:43157 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b543b351fe95,43157,1733463596184' 2024-12-06T05:39:57,132 DEBUG [RS:1;b543b351fe95:43157 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T05:39:57,132 DEBUG [RS:1;b543b351fe95:43157 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T05:39:57,132 DEBUG [RS:1;b543b351fe95:43157 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T05:39:57,132 INFO [RS:1;b543b351fe95:43157 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T05:39:57,132 INFO [RS:1;b543b351fe95:43157 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T05:39:57,200 INFO [RS:0;b543b351fe95:42121 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:39:57,202 INFO [RS:0;b543b351fe95:42121 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b543b351fe95%2C42121%2C1733463596109, suffix=, logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,42121,1733463596109, archiveDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/oldWALs, maxLogs=32 2024-12-06T05:39:57,219 DEBUG [RS:0;b543b351fe95:42121 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,42121,1733463596109/b543b351fe95%2C42121%2C1733463596109.1733463597202, exclude list is [], retry=0 2024-12-06T05:39:57,223 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39561,DS-4485eaa6-367b-4084-97e1-b834fb8d8596,DISK] 2024-12-06T05:39:57,223 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-80ddb16c-ec66-4e39-a44c-0a80fa143251,DISK] 2024-12-06T05:39:57,228 INFO [RS:0;b543b351fe95:42121 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,42121,1733463596109/b543b351fe95%2C42121%2C1733463596109.1733463597202 2024-12-06T05:39:57,228 DEBUG [RS:0;b543b351fe95:42121 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43559:43559),(127.0.0.1/127.0.0.1:44855:44855)] 2024-12-06T05:39:57,233 INFO [RS:1;b543b351fe95:43157 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:39:57,235 INFO [RS:1;b543b351fe95:43157 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b543b351fe95%2C43157%2C1733463596184, suffix=, logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,43157,1733463596184, archiveDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/oldWALs, maxLogs=32 2024-12-06T05:39:57,248 DEBUG [RS:1;b543b351fe95:43157 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,43157,1733463596184/b543b351fe95%2C43157%2C1733463596184.1733463597235, exclude list is [], retry=0 2024-12-06T05:39:57,254 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-80ddb16c-ec66-4e39-a44c-0a80fa143251,DISK] 2024-12-06T05:39:57,254 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39561,DS-4485eaa6-367b-4084-97e1-b834fb8d8596,DISK] 2024-12-06T05:39:57,257 INFO [RS:1;b543b351fe95:43157 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,43157,1733463596184/b543b351fe95%2C43157%2C1733463596184.1733463597235 2024-12-06T05:39:57,257 DEBUG [RS:1;b543b351fe95:43157 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43559:43559),(127.0.0.1/127.0.0.1:44855:44855)] 2024-12-06T05:39:57,378 DEBUG [b543b351fe95:42019 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-06T05:39:57,379 DEBUG [b543b351fe95:42019 {}] balancer.BalancerClusterState(204): Hosts are {b543b351fe95=0} racks are {/default-rack=0} 2024-12-06T05:39:57,381 DEBUG [b543b351fe95:42019 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-06T05:39:57,381 DEBUG [b543b351fe95:42019 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-06T05:39:57,381 DEBUG [b543b351fe95:42019 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-06T05:39:57,381 DEBUG [b543b351fe95:42019 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-06T05:39:57,381 INFO [b543b351fe95:42019 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-06T05:39:57,381 INFO [b543b351fe95:42019 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-06T05:39:57,381 DEBUG [b543b351fe95:42019 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-06T05:39:57,381 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b543b351fe95,42121,1733463596109 2024-12-06T05:39:57,383 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b543b351fe95,42121,1733463596109, state=OPENING 2024-12-06T05:39:57,384 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-06T05:39:57,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:57,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:57,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43157-0x100ae0a5c730015, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:57,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42121-0x100ae0a5c730014, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:57,386 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:57,389 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:57,389 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:57,389 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:57,390 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T05:39:57,390 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b543b351fe95,42121,1733463596109}] 2024-12-06T05:39:57,550 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T05:39:57,558 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36277, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T05:39:57,573 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-06T05:39:57,574 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:39:57,579 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-06T05:39:57,587 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b543b351fe95%2C42121%2C1733463596109.meta, suffix=.meta, logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,42121,1733463596109, archiveDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/oldWALs, maxLogs=32 2024-12-06T05:39:57,608 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,42121,1733463596109/b543b351fe95%2C42121%2C1733463596109.meta.1733463597588.meta, exclude list is [], retry=0 2024-12-06T05:39:57,614 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39561,DS-4485eaa6-367b-4084-97e1-b834fb8d8596,DISK] 2024-12-06T05:39:57,615 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-80ddb16c-ec66-4e39-a44c-0a80fa143251,DISK] 2024-12-06T05:39:57,631 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,42121,1733463596109/b543b351fe95%2C42121%2C1733463596109.meta.1733463597588.meta 2024-12-06T05:39:57,633 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43559:43559),(127.0.0.1/127.0.0.1:44855:44855)] 2024-12-06T05:39:57,635 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:39:57,636 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-06T05:39:57,636 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-06T05:39:57,636 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-06T05:39:57,637 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-06T05:39:57,637 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:57,637 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-06T05:39:57,638 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-06T05:39:57,650 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-06T05:39:57,659 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T05:39:57,659 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:57,667 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:57,667 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-06T05:39:57,669 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T05:39:57,669 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:57,669 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:57,670 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-06T05:39:57,672 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T05:39:57,672 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:57,674 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:57,675 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-06T05:39:57,677 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T05:39:57,677 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:57,678 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:57,678 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T05:39:57,680 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/hbase/meta/1588230740 2024-12-06T05:39:57,681 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/hbase/meta/1588230740 2024-12-06T05:39:57,684 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T05:39:57,684 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T05:39:57,686 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T05:39:57,689 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T05:39:57,693 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72072565, jitterRate=0.0739649087190628}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T05:39:57,693 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-06T05:39:57,694 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733463597641Writing region info on filesystem at 1733463597641Initializing all the Stores at 1733463597643 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463597643Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463597650 (+7 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463597650Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463597650Cleaning up temporary data from old regions at 1733463597684 (+34 ms)Running coprocessor post-open hooks at 1733463597693 (+9 ms)Region opened successfully at 1733463597694 (+1 ms) 2024-12-06T05:39:57,695 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733463597549 2024-12-06T05:39:57,698 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-06T05:39:57,698 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-06T05:39:57,703 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b543b351fe95,42121,1733463596109 2024-12-06T05:39:57,708 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b543b351fe95,42121,1733463596109, state=OPEN 2024-12-06T05:39:57,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42121-0x100ae0a5c730014, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:39:57,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:39:57,710 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:57,710 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:57,710 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b543b351fe95,42121,1733463596109 2024-12-06T05:39:57,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:39:57,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43157-0x100ae0a5c730015, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:39:57,710 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:57,710 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:57,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-06T05:39:57,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b543b351fe95,42121,1733463596109 in 320 msec 2024-12-06T05:39:57,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-06T05:39:57,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 740 msec 2024-12-06T05:39:57,721 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T05:39:57,721 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-06T05:39:57,724 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T05:39:57,725 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b543b351fe95,42121,1733463596109, seqNum=-1] 2024-12-06T05:39:57,725 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T05:39:57,738 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48545, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T05:39:57,769 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 968 msec 2024-12-06T05:39:57,769 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733463597769, completionTime=-1 2024-12-06T05:39:57,769 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-06T05:39:57,769 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-06T05:39:57,774 INFO [master/b543b351fe95:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-06T05:39:57,774 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733463657774 2024-12-06T05:39:57,774 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733463717774 2024-12-06T05:39:57,774 INFO [master/b543b351fe95:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec 2024-12-06T05:39:57,774 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-06T05:39:57,777 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,42019,1733463595907-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,777 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,42019,1733463595907-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,777 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,42019,1733463595907-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,777 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b543b351fe95:42019, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,777 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,780 DEBUG [master/b543b351fe95:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-06T05:39:57,789 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,794 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.487sec 2024-12-06T05:39:57,799 INFO [master/b543b351fe95:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-06T05:39:57,800 INFO [master/b543b351fe95:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-06T05:39:57,800 INFO [master/b543b351fe95:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-06T05:39:57,800 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-06T05:39:57,800 INFO [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-06T05:39:57,800 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,42019,1733463595907-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T05:39:57,800 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,42019,1733463595907-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-06T05:39:57,805 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-06T05:39:57,805 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-06T05:39:57,805 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,42019,1733463595907-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,840 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6af7b1db to 127.0.0.1:58086 with session timeout=1000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5778c73a 2024-12-06T05:39:57,854 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@629deb0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T05:39:57,855 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T05:39:57,857 DEBUG [ReadOnlyZKClient-127.0.0.1:58086@0x6af7b1db {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b543b351fe95,42121,1733463596109, seqNum=-1] 2024-12-06T05:39:57,858 DEBUG [ReadOnlyZKClient-127.0.0.1:58086@0x6af7b1db {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T05:39:57,860 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T05:39:57,862 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=b543b351fe95,42019,1733463595907 2024-12-06T05:39:57,883 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b543b351fe95:0 server-side Connection retries=45 2024-12-06T05:39:57,883 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:57,883 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:57,883 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T05:39:57,883 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:39:57,883 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T05:39:57,883 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T05:39:57,883 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T05:39:57,884 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44975 2024-12-06T05:39:57,885 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44975 connecting to ZooKeeper ensemble=127.0.0.1:58086 2024-12-06T05:39:57,886 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:57,889 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:57,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:449750x0, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:39:57,898 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44975-0x100ae0a5c730017 connected 2024-12-06T05:39:57,901 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:44975-0x100ae0a5c730017, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-06T05:39:57,901 DEBUG [pool-708-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-06T05:39:57,902 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T05:39:57,904 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T05:39:57,905 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:44975-0x100ae0a5c730017, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T05:39:57,907 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44975-0x100ae0a5c730017, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T05:39:57,908 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44975 2024-12-06T05:39:57,908 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44975 2024-12-06T05:39:57,909 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44975 2024-12-06T05:39:57,909 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44975 2024-12-06T05:39:57,909 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44975 2024-12-06T05:39:57,921 INFO [RS:2;b543b351fe95:44975 {}] regionserver.HRegionServer(746): ClusterId : b7c25f36-05b2-43f3-92e5-304406a0c4c1 2024-12-06T05:39:57,921 DEBUG [RS:2;b543b351fe95:44975 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T05:39:57,923 DEBUG [RS:2;b543b351fe95:44975 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T05:39:57,923 DEBUG [RS:2;b543b351fe95:44975 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T05:39:57,925 DEBUG [RS:2;b543b351fe95:44975 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T05:39:57,925 DEBUG [RS:2;b543b351fe95:44975 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d2a5167, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b543b351fe95/172.17.0.2:0 2024-12-06T05:39:57,941 DEBUG [RS:2;b543b351fe95:44975 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;b543b351fe95:44975 2024-12-06T05:39:57,941 INFO [RS:2;b543b351fe95:44975 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T05:39:57,941 INFO [RS:2;b543b351fe95:44975 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T05:39:57,942 DEBUG [RS:2;b543b351fe95:44975 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T05:39:57,942 INFO [RS:2;b543b351fe95:44975 {}] regionserver.HRegionServer(2659): reportForDuty to master=b543b351fe95,42019,1733463595907 with port=44975, startcode=1733463597882 2024-12-06T05:39:57,942 DEBUG [RS:2;b543b351fe95:44975 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T05:39:57,946 INFO [HMaster-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43949, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.8 (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T05:39:57,946 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42019 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b543b351fe95,44975,1733463597882 2024-12-06T05:39:57,946 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42019 {}] master.ServerManager(517): Registering regionserver=b543b351fe95,44975,1733463597882 2024-12-06T05:39:57,948 DEBUG [RS:2;b543b351fe95:44975 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:39:57,948 DEBUG [RS:2;b543b351fe95:44975 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34453 2024-12-06T05:39:57,948 DEBUG [RS:2;b543b351fe95:44975 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T05:39:57,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T05:39:57,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T05:39:57,951 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b543b351fe95,44975,1733463597882] 2024-12-06T05:39:57,951 DEBUG [RS:2;b543b351fe95:44975 {}] zookeeper.ZKUtil(111): regionserver:44975-0x100ae0a5c730017, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b543b351fe95,44975,1733463597882 2024-12-06T05:39:57,951 WARN [RS:2;b543b351fe95:44975 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T05:39:57,951 INFO [RS:2;b543b351fe95:44975 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:39:57,952 DEBUG [RS:2;b543b351fe95:44975 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,44975,1733463597882 2024-12-06T05:39:57,956 INFO [RS:2;b543b351fe95:44975 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T05:39:57,961 INFO [RS:2;b543b351fe95:44975 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T05:39:57,962 INFO [RS:2;b543b351fe95:44975 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T05:39:57,962 INFO [RS:2;b543b351fe95:44975 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,963 INFO [RS:2;b543b351fe95:44975 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T05:39:57,964 INFO [RS:2;b543b351fe95:44975 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T05:39:57,964 INFO [RS:2;b543b351fe95:44975 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,964 DEBUG [RS:2;b543b351fe95:44975 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,964 DEBUG [RS:2;b543b351fe95:44975 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,964 DEBUG [RS:2;b543b351fe95:44975 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,964 DEBUG [RS:2;b543b351fe95:44975 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,964 DEBUG [RS:2;b543b351fe95:44975 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,964 DEBUG [RS:2;b543b351fe95:44975 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0, corePoolSize=2, maxPoolSize=2 2024-12-06T05:39:57,964 DEBUG [RS:2;b543b351fe95:44975 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,964 DEBUG [RS:2;b543b351fe95:44975 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,964 DEBUG [RS:2;b543b351fe95:44975 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,964 DEBUG [RS:2;b543b351fe95:44975 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,964 DEBUG [RS:2;b543b351fe95:44975 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,964 DEBUG [RS:2;b543b351fe95:44975 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:57,964 DEBUG [RS:2;b543b351fe95:44975 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b543b351fe95:0, corePoolSize=3, maxPoolSize=3 2024-12-06T05:39:57,964 DEBUG [RS:2;b543b351fe95:44975 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b543b351fe95:0, corePoolSize=3, maxPoolSize=3 2024-12-06T05:39:57,965 INFO [RS:2;b543b351fe95:44975 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,965 INFO [RS:2;b543b351fe95:44975 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,965 INFO [RS:2;b543b351fe95:44975 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,965 INFO [RS:2;b543b351fe95:44975 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,965 INFO [RS:2;b543b351fe95:44975 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,965 INFO [RS:2;b543b351fe95:44975 {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,44975,1733463597882-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T05:39:57,980 INFO [RS:2;b543b351fe95:44975 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T05:39:57,981 INFO [RS:2;b543b351fe95:44975 {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,44975,1733463597882-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,981 INFO [RS:2;b543b351fe95:44975 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:57,981 INFO [RS:2;b543b351fe95:44975 {}] regionserver.Replication(171): b543b351fe95,44975,1733463597882 started 2024-12-06T05:39:58,000 INFO [RS:2;b543b351fe95:44975 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:58,000 INFO [RS:2;b543b351fe95:44975 {}] regionserver.HRegionServer(1482): Serving as b543b351fe95,44975,1733463597882, RpcServer on b543b351fe95/172.17.0.2:44975, sessionid=0x100ae0a5c730017 2024-12-06T05:39:58,000 DEBUG [RS:2;b543b351fe95:44975 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T05:39:58,000 DEBUG [RS:2;b543b351fe95:44975 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b543b351fe95,44975,1733463597882 2024-12-06T05:39:58,000 DEBUG [RS:2;b543b351fe95:44975 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b543b351fe95,44975,1733463597882' 2024-12-06T05:39:58,000 DEBUG [RS:2;b543b351fe95:44975 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T05:39:58,000 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T05:39:58,001 DEBUG [RS:2;b543b351fe95:44975 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T05:39:58,001 DEBUG [RS:2;b543b351fe95:44975 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T05:39:58,001 DEBUG [RS:2;b543b351fe95:44975 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T05:39:58,001 DEBUG [RS:2;b543b351fe95:44975 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b543b351fe95,44975,1733463597882 2024-12-06T05:39:58,001 DEBUG [RS:2;b543b351fe95:44975 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b543b351fe95,44975,1733463597882' 2024-12-06T05:39:58,001 DEBUG [RS:2;b543b351fe95:44975 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T05:39:58,001 DEBUG [ReadOnlyZKClient-127.0.0.1:58086@0x6af7b1db {}] client.AsyncConnectionImpl(321): The fetched master address is b543b351fe95,42019,1733463595907 2024-12-06T05:39:58,001 DEBUG [ReadOnlyZKClient-127.0.0.1:58086@0x6af7b1db {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@535d1ebb 2024-12-06T05:39:58,002 DEBUG [ReadOnlyZKClient-127.0.0.1:58086@0x6af7b1db {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T05:39:58,002 DEBUG [RS:2;b543b351fe95:44975 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T05:39:58,002 DEBUG [RS:2;b543b351fe95:44975 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T05:39:58,002 INFO [RS:2;b543b351fe95:44975 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T05:39:58,002 INFO [RS:2;b543b351fe95:44975 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T05:39:58,004 INFO [HMaster-EventLoopGroup-15-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T05:39:58,005 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testLogSplittingAfterMasterRecoveryDueToZKExpiry', {NAME => 'col', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T05:39:58,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42019 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry 2024-12-06T05:39:58,008 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T05:39:58,008 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:58,008 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testLogSplittingAfterMasterRecoveryDueToZKExpiry" procId is: 4 2024-12-06T05:39:58,010 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T05:39:58,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T05:39:58,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741890_1067 (size=399) 2024-12-06T05:39:58,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741890_1067 (size=399) 2024-12-06T05:39:58,024 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4e5ef5686145b3308e84acf2485afdef, NAME => 'testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef.', STARTKEY => '', ENDKEY => '1'}, tableDescriptor='testLogSplittingAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'col', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:39:58,030 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 59e2fa4009ebd132f92d2a7d7e292ba0, NAME => 'testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1733463598004.59e2fa4009ebd132f92d2a7d7e292ba0.', STARTKEY => '1', ENDKEY => '2'}, tableDescriptor='testLogSplittingAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'col', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:39:58,030 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => ad0165368b2f0ee58b25d3d51e5286d5, NAME => 'testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1733463598004.ad0165368b2f0ee58b25d3d51e5286d5.', STARTKEY => '2', ENDKEY => '3'}, tableDescriptor='testLogSplittingAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'col', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:39:58,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741891_1068 (size=84) 2024-12-06T05:39:58,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741891_1068 (size=84) 2024-12-06T05:39:58,054 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(898): Instantiated testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:58,054 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1722): Closing 4e5ef5686145b3308e84acf2485afdef, disabling compactions & flushes 2024-12-06T05:39:58,054 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1755): Closing region testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef. 2024-12-06T05:39:58,054 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef. 2024-12-06T05:39:58,054 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef. after waiting 0 ms 2024-12-06T05:39:58,054 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef. 2024-12-06T05:39:58,054 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1973): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef. 2024-12-06T05:39:58,054 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4e5ef5686145b3308e84acf2485afdef: Waiting for close lock at 1733463598054Disabling compacts and flushes for region at 1733463598054Disabling writes for close at 1733463598054Writing region close event to WAL at 1733463598054Closed at 1733463598054 2024-12-06T05:39:58,055 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 71150a984937a6ec0a3a8eb3ffafae8c, NAME => 'testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1733463598004.71150a984937a6ec0a3a8eb3ffafae8c.', STARTKEY => '3', ENDKEY => '4'}, tableDescriptor='testLogSplittingAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'col', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:39:58,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741892_1069 (size=85) 2024-12-06T05:39:58,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741892_1069 (size=85) 2024-12-06T05:39:58,059 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(898): Instantiated testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1733463598004.ad0165368b2f0ee58b25d3d51e5286d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:58,060 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1722): Closing ad0165368b2f0ee58b25d3d51e5286d5, disabling compactions & flushes 2024-12-06T05:39:58,060 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1755): Closing region testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1733463598004.ad0165368b2f0ee58b25d3d51e5286d5. 2024-12-06T05:39:58,060 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1733463598004.ad0165368b2f0ee58b25d3d51e5286d5. 2024-12-06T05:39:58,060 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1733463598004.ad0165368b2f0ee58b25d3d51e5286d5. after waiting 0 ms 2024-12-06T05:39:58,060 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1733463598004.ad0165368b2f0ee58b25d3d51e5286d5. 2024-12-06T05:39:58,060 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1973): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1733463598004.ad0165368b2f0ee58b25d3d51e5286d5. 2024-12-06T05:39:58,060 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1676): Region close journal for ad0165368b2f0ee58b25d3d51e5286d5: Waiting for close lock at 1733463598060Disabling compacts and flushes for region at 1733463598060Disabling writes for close at 1733463598060Writing region close event to WAL at 1733463598060Closed at 1733463598060 2024-12-06T05:39:58,061 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 91b02cf9f147172bb99e96b9a5ebc6c0, NAME => 'testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1733463598004.91b02cf9f147172bb99e96b9a5ebc6c0.', STARTKEY => '4', ENDKEY => '5'}, tableDescriptor='testLogSplittingAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'col', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:39:58,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741893_1070 (size=85) 2024-12-06T05:39:58,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741893_1070 (size=85) 2024-12-06T05:39:58,073 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(898): Instantiated testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1733463598004.59e2fa4009ebd132f92d2a7d7e292ba0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:58,073 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1722): Closing 59e2fa4009ebd132f92d2a7d7e292ba0, disabling compactions & flushes 2024-12-06T05:39:58,073 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1755): Closing region testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1733463598004.59e2fa4009ebd132f92d2a7d7e292ba0. 2024-12-06T05:39:58,073 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1733463598004.59e2fa4009ebd132f92d2a7d7e292ba0. 2024-12-06T05:39:58,073 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1733463598004.59e2fa4009ebd132f92d2a7d7e292ba0. after waiting 0 ms 2024-12-06T05:39:58,073 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1733463598004.59e2fa4009ebd132f92d2a7d7e292ba0. 2024-12-06T05:39:58,073 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1973): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1733463598004.59e2fa4009ebd132f92d2a7d7e292ba0. 2024-12-06T05:39:58,073 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1676): Region close journal for 59e2fa4009ebd132f92d2a7d7e292ba0: Waiting for close lock at 1733463598073Disabling compacts and flushes for region at 1733463598073Disabling writes for close at 1733463598073Writing region close event to WAL at 1733463598073Closed at 1733463598073 2024-12-06T05:39:58,074 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => f711afceea9ea8582107efd7e8aa74eb, NAME => 'testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1733463598004.f711afceea9ea8582107efd7e8aa74eb.', STARTKEY => '5', ENDKEY => ''}, tableDescriptor='testLogSplittingAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'col', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:39:58,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741894_1071 (size=85) 2024-12-06T05:39:58,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741894_1071 (size=85) 2024-12-06T05:39:58,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741895_1072 (size=85) 2024-12-06T05:39:58,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741895_1072 (size=85) 2024-12-06T05:39:58,086 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(898): Instantiated testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1733463598004.91b02cf9f147172bb99e96b9a5ebc6c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:58,086 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1722): Closing 91b02cf9f147172bb99e96b9a5ebc6c0, disabling compactions & flushes 2024-12-06T05:39:58,086 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1755): Closing region testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1733463598004.91b02cf9f147172bb99e96b9a5ebc6c0. 2024-12-06T05:39:58,087 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1733463598004.91b02cf9f147172bb99e96b9a5ebc6c0. 2024-12-06T05:39:58,087 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1733463598004.91b02cf9f147172bb99e96b9a5ebc6c0. after waiting 0 ms 2024-12-06T05:39:58,087 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1733463598004.91b02cf9f147172bb99e96b9a5ebc6c0. 2024-12-06T05:39:58,087 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1973): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1733463598004.91b02cf9f147172bb99e96b9a5ebc6c0. 2024-12-06T05:39:58,087 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1676): Region close journal for 91b02cf9f147172bb99e96b9a5ebc6c0: Waiting for close lock at 1733463598086Disabling compacts and flushes for region at 1733463598086Disabling writes for close at 1733463598087 (+1 ms)Writing region close event to WAL at 1733463598087Closed at 1733463598087 2024-12-06T05:39:58,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741896_1073 (size=84) 2024-12-06T05:39:58,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741896_1073 (size=84) 2024-12-06T05:39:58,103 INFO [RS:2;b543b351fe95:44975 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:39:58,111 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(898): Instantiated testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1733463598004.f711afceea9ea8582107efd7e8aa74eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:58,111 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1722): Closing f711afceea9ea8582107efd7e8aa74eb, disabling compactions & flushes 2024-12-06T05:39:58,111 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1755): Closing region testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1733463598004.f711afceea9ea8582107efd7e8aa74eb. 2024-12-06T05:39:58,111 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1733463598004.f711afceea9ea8582107efd7e8aa74eb. 2024-12-06T05:39:58,111 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1733463598004.f711afceea9ea8582107efd7e8aa74eb. after waiting 0 ms 2024-12-06T05:39:58,111 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1733463598004.f711afceea9ea8582107efd7e8aa74eb. 2024-12-06T05:39:58,111 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1973): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1733463598004.f711afceea9ea8582107efd7e8aa74eb. 2024-12-06T05:39:58,111 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1676): Region close journal for f711afceea9ea8582107efd7e8aa74eb: Waiting for close lock at 1733463598111Disabling compacts and flushes for region at 1733463598111Disabling writes for close at 1733463598111Writing region close event to WAL at 1733463598111Closed at 1733463598111 2024-12-06T05:39:58,113 INFO [RS:2;b543b351fe95:44975 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b543b351fe95%2C44975%2C1733463597882, suffix=, logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,44975,1733463597882, archiveDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/oldWALs, maxLogs=32 2024-12-06T05:39:58,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T05:39:58,127 DEBUG [RS:2;b543b351fe95:44975 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,44975,1733463597882/b543b351fe95%2C44975%2C1733463597882.1733463598114, exclude list is [], retry=0 2024-12-06T05:39:58,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-19-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-80ddb16c-ec66-4e39-a44c-0a80fa143251,DISK] 2024-12-06T05:39:58,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-19-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39561,DS-4485eaa6-367b-4084-97e1-b834fb8d8596,DISK] 2024-12-06T05:39:58,142 INFO [RS:2;b543b351fe95:44975 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,44975,1733463597882/b543b351fe95%2C44975%2C1733463597882.1733463598114 2024-12-06T05:39:58,142 DEBUG [RS:2;b543b351fe95:44975 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44855:44855),(127.0.0.1/127.0.0.1:43559:43559)] 2024-12-06T05:39:58,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T05:39:58,476 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(898): Instantiated testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1733463598004.71150a984937a6ec0a3a8eb3ffafae8c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:58,477 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1722): Closing 71150a984937a6ec0a3a8eb3ffafae8c, disabling compactions & flushes 2024-12-06T05:39:58,477 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1755): Closing region testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1733463598004.71150a984937a6ec0a3a8eb3ffafae8c. 2024-12-06T05:39:58,477 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1733463598004.71150a984937a6ec0a3a8eb3ffafae8c. 2024-12-06T05:39:58,477 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1733463598004.71150a984937a6ec0a3a8eb3ffafae8c. after waiting 0 ms 2024-12-06T05:39:58,477 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1733463598004.71150a984937a6ec0a3a8eb3ffafae8c. 2024-12-06T05:39:58,477 INFO [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1973): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1733463598004.71150a984937a6ec0a3a8eb3ffafae8c. 2024-12-06T05:39:58,477 DEBUG [RegionOpenAndInit-testLogSplittingAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1676): Region close journal for 71150a984937a6ec0a3a8eb3ffafae8c: Waiting for close lock at 1733463598477Disabling compacts and flushes for region at 1733463598477Disabling writes for close at 1733463598477Writing region close event to WAL at 1733463598477Closed at 1733463598477 2024-12-06T05:39:58,481 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T05:39:58,482 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef.","families":{"info":[{"qualifier":"regioninfo","vlen":83,"tag":[],"timestamp":"1733463598481"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733463598481"}]},"ts":"1733463598481"} 2024-12-06T05:39:58,482 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1733463598004.ad0165368b2f0ee58b25d3d51e5286d5.","families":{"info":[{"qualifier":"regioninfo","vlen":84,"tag":[],"timestamp":"1733463598481"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733463598481"}]},"ts":"1733463598481"} 2024-12-06T05:39:58,482 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1733463598004.59e2fa4009ebd132f92d2a7d7e292ba0.","families":{"info":[{"qualifier":"regioninfo","vlen":84,"tag":[],"timestamp":"1733463598481"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733463598481"}]},"ts":"1733463598481"} 2024-12-06T05:39:58,483 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1733463598004.91b02cf9f147172bb99e96b9a5ebc6c0.","families":{"info":[{"qualifier":"regioninfo","vlen":84,"tag":[],"timestamp":"1733463598481"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733463598481"}]},"ts":"1733463598481"} 2024-12-06T05:39:58,483 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1733463598004.f711afceea9ea8582107efd7e8aa74eb.","families":{"info":[{"qualifier":"regioninfo","vlen":83,"tag":[],"timestamp":"1733463598481"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733463598481"}]},"ts":"1733463598481"} 2024-12-06T05:39:58,483 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1733463598004.71150a984937a6ec0a3a8eb3ffafae8c.","families":{"info":[{"qualifier":"regioninfo","vlen":84,"tag":[],"timestamp":"1733463598481"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733463598481"}]},"ts":"1733463598481"} 2024-12-06T05:39:58,525 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 6 regions to meta. 2024-12-06T05:39:58,527 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T05:39:58,527 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testLogSplittingAfterMasterRecoveryDueToZKExpiry","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733463598527"}]},"ts":"1733463598527"} 2024-12-06T05:39:58,530 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testLogSplittingAfterMasterRecoveryDueToZKExpiry, state=ENABLING in hbase:meta 2024-12-06T05:39:58,530 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {b543b351fe95=0} racks are {/default-rack=0} 2024-12-06T05:39:58,544 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-06T05:39:58,544 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-06T05:39:58,544 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-06T05:39:58,544 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-06T05:39:58,544 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-06T05:39:58,544 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-06T05:39:58,545 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-06T05:39:58,545 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-06T05:39:58,545 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-06T05:39:58,545 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-06T05:39:58,550 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=4e5ef5686145b3308e84acf2485afdef, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=59e2fa4009ebd132f92d2a7d7e292ba0, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=ad0165368b2f0ee58b25d3d51e5286d5, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=71150a984937a6ec0a3a8eb3ffafae8c, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=91b02cf9f147172bb99e96b9a5ebc6c0, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=f711afceea9ea8582107efd7e8aa74eb, ASSIGN}] 2024-12-06T05:39:58,558 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=4e5ef5686145b3308e84acf2485afdef, ASSIGN 2024-12-06T05:39:58,560 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=4e5ef5686145b3308e84acf2485afdef, ASSIGN; state=OFFLINE, location=b543b351fe95,44975,1733463597882; forceNewPlan=false, retain=false 2024-12-06T05:39:58,560 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=59e2fa4009ebd132f92d2a7d7e292ba0, ASSIGN 2024-12-06T05:39:58,563 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=91b02cf9f147172bb99e96b9a5ebc6c0, ASSIGN 2024-12-06T05:39:58,563 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=59e2fa4009ebd132f92d2a7d7e292ba0, ASSIGN; state=OFFLINE, location=b543b351fe95,42121,1733463596109; forceNewPlan=false, retain=false 2024-12-06T05:39:58,563 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=ad0165368b2f0ee58b25d3d51e5286d5, ASSIGN 2024-12-06T05:39:58,564 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=f711afceea9ea8582107efd7e8aa74eb, ASSIGN 2024-12-06T05:39:58,568 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=ad0165368b2f0ee58b25d3d51e5286d5, ASSIGN; state=OFFLINE, location=b543b351fe95,43157,1733463596184; forceNewPlan=false, retain=false 2024-12-06T05:39:58,568 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=91b02cf9f147172bb99e96b9a5ebc6c0, ASSIGN; state=OFFLINE, location=b543b351fe95,44975,1733463597882; forceNewPlan=false, retain=false 2024-12-06T05:39:58,568 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=f711afceea9ea8582107efd7e8aa74eb, ASSIGN; state=OFFLINE, location=b543b351fe95,43157,1733463596184; forceNewPlan=false, retain=false 2024-12-06T05:39:58,568 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=71150a984937a6ec0a3a8eb3ffafae8c, ASSIGN 2024-12-06T05:39:58,570 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=71150a984937a6ec0a3a8eb3ffafae8c, ASSIGN; state=OFFLINE, location=b543b351fe95,42121,1733463596109; forceNewPlan=false, retain=false 2024-12-06T05:39:58,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T05:39:58,711 INFO [b543b351fe95:42019 {}] balancer.BaseLoadBalancer(388): Reassigned 6 regions. 6 retained the pre-restart assignment. 2024-12-06T05:39:58,711 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=91b02cf9f147172bb99e96b9a5ebc6c0, regionState=OPENING, regionLocation=b543b351fe95,44975,1733463597882 2024-12-06T05:39:58,711 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4e5ef5686145b3308e84acf2485afdef, regionState=OPENING, regionLocation=b543b351fe95,44975,1733463597882 2024-12-06T05:39:58,712 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=f711afceea9ea8582107efd7e8aa74eb, regionState=OPENING, regionLocation=b543b351fe95,43157,1733463596184 2024-12-06T05:39:58,712 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ad0165368b2f0ee58b25d3d51e5286d5, regionState=OPENING, regionLocation=b543b351fe95,43157,1733463596184 2024-12-06T05:39:58,712 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=71150a984937a6ec0a3a8eb3ffafae8c, regionState=OPENING, regionLocation=b543b351fe95,42121,1733463596109 2024-12-06T05:39:58,713 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=59e2fa4009ebd132f92d2a7d7e292ba0, regionState=OPENING, regionLocation=b543b351fe95,42121,1733463596109 2024-12-06T05:39:58,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=91b02cf9f147172bb99e96b9a5ebc6c0, ASSIGN because future has completed 2024-12-06T05:39:58,718 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 91b02cf9f147172bb99e96b9a5ebc6c0, server=b543b351fe95,44975,1733463597882}] 2024-12-06T05:39:58,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=4e5ef5686145b3308e84acf2485afdef, ASSIGN because future has completed 2024-12-06T05:39:58,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e5ef5686145b3308e84acf2485afdef, server=b543b351fe95,44975,1733463597882}] 2024-12-06T05:39:58,726 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=ad0165368b2f0ee58b25d3d51e5286d5, ASSIGN because future has completed 2024-12-06T05:39:58,729 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure ad0165368b2f0ee58b25d3d51e5286d5, server=b543b351fe95,43157,1733463596184}] 2024-12-06T05:39:58,730 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=f711afceea9ea8582107efd7e8aa74eb, ASSIGN because future has completed 2024-12-06T05:39:58,733 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=71150a984937a6ec0a3a8eb3ffafae8c, ASSIGN because future has completed 2024-12-06T05:39:58,746 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure f711afceea9ea8582107efd7e8aa74eb, server=b543b351fe95,43157,1733463596184}] 2024-12-06T05:39:58,746 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=59e2fa4009ebd132f92d2a7d7e292ba0, ASSIGN because future has completed 2024-12-06T05:39:58,750 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59e2fa4009ebd132f92d2a7d7e292ba0, server=b543b351fe95,42121,1733463596109}] 2024-12-06T05:39:58,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 71150a984937a6ec0a3a8eb3ffafae8c, server=b543b351fe95,42121,1733463596109}] 2024-12-06T05:39:58,875 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T05:39:58,877 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-19-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48335, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T05:39:58,883 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1733463598004.91b02cf9f147172bb99e96b9a5ebc6c0. 2024-12-06T05:39:58,884 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 91b02cf9f147172bb99e96b9a5ebc6c0, NAME => 'testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1733463598004.91b02cf9f147172bb99e96b9a5ebc6c0.', STARTKEY => '4', ENDKEY => '5'} 2024-12-06T05:39:58,884 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testLogSplittingAfterMasterRecoveryDueToZKExpiry 91b02cf9f147172bb99e96b9a5ebc6c0 2024-12-06T05:39:58,884 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1733463598004.91b02cf9f147172bb99e96b9a5ebc6c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:58,884 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 91b02cf9f147172bb99e96b9a5ebc6c0 2024-12-06T05:39:58,884 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 91b02cf9f147172bb99e96b9a5ebc6c0 2024-12-06T05:39:58,901 INFO [StoreOpener-91b02cf9f147172bb99e96b9a5ebc6c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family col of region 91b02cf9f147172bb99e96b9a5ebc6c0 2024-12-06T05:39:58,903 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T05:39:58,904 INFO [StoreOpener-91b02cf9f147172bb99e96b9a5ebc6c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 91b02cf9f147172bb99e96b9a5ebc6c0 columnFamilyName col 2024-12-06T05:39:58,904 DEBUG [StoreOpener-91b02cf9f147172bb99e96b9a5ebc6c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:58,904 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59247, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T05:39:58,904 INFO [StoreOpener-91b02cf9f147172bb99e96b9a5ebc6c0-1 {}] regionserver.HStore(327): Store=91b02cf9f147172bb99e96b9a5ebc6c0/col, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:39:58,905 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 91b02cf9f147172bb99e96b9a5ebc6c0 2024-12-06T05:39:58,906 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/91b02cf9f147172bb99e96b9a5ebc6c0 2024-12-06T05:39:58,906 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/91b02cf9f147172bb99e96b9a5ebc6c0 2024-12-06T05:39:58,909 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 91b02cf9f147172bb99e96b9a5ebc6c0 2024-12-06T05:39:58,910 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 91b02cf9f147172bb99e96b9a5ebc6c0 2024-12-06T05:39:58,911 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] handler.AssignRegionHandler(132): Open testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1733463598004.f711afceea9ea8582107efd7e8aa74eb. 2024-12-06T05:39:58,911 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(7752): Opening region: {ENCODED => f711afceea9ea8582107efd7e8aa74eb, NAME => 'testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1733463598004.f711afceea9ea8582107efd7e8aa74eb.', STARTKEY => '5', ENDKEY => ''} 2024-12-06T05:39:58,911 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testLogSplittingAfterMasterRecoveryDueToZKExpiry f711afceea9ea8582107efd7e8aa74eb 2024-12-06T05:39:58,911 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(898): Instantiated testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1733463598004.f711afceea9ea8582107efd7e8aa74eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:58,911 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(7794): checking encryption for f711afceea9ea8582107efd7e8aa74eb 2024-12-06T05:39:58,912 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(7797): checking classloading for f711afceea9ea8582107efd7e8aa74eb 2024-12-06T05:39:58,912 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(132): Open testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1733463598004.71150a984937a6ec0a3a8eb3ffafae8c. 2024-12-06T05:39:58,912 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7752): Opening region: {ENCODED => 71150a984937a6ec0a3a8eb3ffafae8c, NAME => 'testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1733463598004.71150a984937a6ec0a3a8eb3ffafae8c.', STARTKEY => '3', ENDKEY => '4'} 2024-12-06T05:39:58,915 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testLogSplittingAfterMasterRecoveryDueToZKExpiry 71150a984937a6ec0a3a8eb3ffafae8c 2024-12-06T05:39:58,915 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(898): Instantiated testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1733463598004.71150a984937a6ec0a3a8eb3ffafae8c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:58,916 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7794): checking encryption for 71150a984937a6ec0a3a8eb3ffafae8c 2024-12-06T05:39:58,916 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7797): checking classloading for 71150a984937a6ec0a3a8eb3ffafae8c 2024-12-06T05:39:58,916 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 91b02cf9f147172bb99e96b9a5ebc6c0 2024-12-06T05:39:58,921 INFO [StoreOpener-71150a984937a6ec0a3a8eb3ffafae8c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family col of region 71150a984937a6ec0a3a8eb3ffafae8c 2024-12-06T05:39:58,922 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/91b02cf9f147172bb99e96b9a5ebc6c0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:39:58,923 INFO [StoreOpener-71150a984937a6ec0a3a8eb3ffafae8c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 71150a984937a6ec0a3a8eb3ffafae8c columnFamilyName col 2024-12-06T05:39:58,923 DEBUG [StoreOpener-71150a984937a6ec0a3a8eb3ffafae8c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:58,923 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 91b02cf9f147172bb99e96b9a5ebc6c0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64453358, jitterRate=-0.039570122957229614}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:39:58,924 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 91b02cf9f147172bb99e96b9a5ebc6c0 2024-12-06T05:39:58,924 INFO [StoreOpener-71150a984937a6ec0a3a8eb3ffafae8c-1 {}] regionserver.HStore(327): Store=71150a984937a6ec0a3a8eb3ffafae8c/col, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:39:58,924 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 91b02cf9f147172bb99e96b9a5ebc6c0: Running coprocessor pre-open hook at 1733463598884Writing region info on filesystem at 1733463598884Initializing all the Stores at 1733463598886 (+2 ms)Instantiating store for column family {NAME => 'col', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463598886Cleaning up temporary data from old regions at 1733463598910 (+24 ms)Running coprocessor post-open hooks at 1733463598924 (+14 ms)Region opened successfully at 1733463598924 2024-12-06T05:39:58,925 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1038): replaying wal for 71150a984937a6ec0a3a8eb3ffafae8c 2024-12-06T05:39:58,925 INFO [StoreOpener-f711afceea9ea8582107efd7e8aa74eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family col of region f711afceea9ea8582107efd7e8aa74eb 2024-12-06T05:39:58,926 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1733463598004.91b02cf9f147172bb99e96b9a5ebc6c0., pid=11, masterSystemTime=1733463598875 2024-12-06T05:39:58,926 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/71150a984937a6ec0a3a8eb3ffafae8c 2024-12-06T05:39:58,927 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/71150a984937a6ec0a3a8eb3ffafae8c 2024-12-06T05:39:58,929 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1048): stopping wal replay for 71150a984937a6ec0a3a8eb3ffafae8c 2024-12-06T05:39:58,929 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1733463598004.91b02cf9f147172bb99e96b9a5ebc6c0. 2024-12-06T05:39:58,929 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1060): Cleaning up temporary data for 71150a984937a6ec0a3a8eb3ffafae8c 2024-12-06T05:39:58,929 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1733463598004.91b02cf9f147172bb99e96b9a5ebc6c0. 2024-12-06T05:39:58,929 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef. 2024-12-06T05:39:58,929 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 4e5ef5686145b3308e84acf2485afdef, NAME => 'testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef.', STARTKEY => '', ENDKEY => '1'} 2024-12-06T05:39:58,930 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testLogSplittingAfterMasterRecoveryDueToZKExpiry 4e5ef5686145b3308e84acf2485afdef 2024-12-06T05:39:58,930 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:58,930 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 4e5ef5686145b3308e84acf2485afdef 2024-12-06T05:39:58,930 INFO [StoreOpener-f711afceea9ea8582107efd7e8aa74eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f711afceea9ea8582107efd7e8aa74eb columnFamilyName col 2024-12-06T05:39:58,930 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 4e5ef5686145b3308e84acf2485afdef 2024-12-06T05:39:58,930 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=91b02cf9f147172bb99e96b9a5ebc6c0, regionState=OPEN, openSeqNum=2, regionLocation=b543b351fe95,44975,1733463597882 2024-12-06T05:39:58,933 DEBUG [StoreOpener-f711afceea9ea8582107efd7e8aa74eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:58,933 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1093): writing seq id for 71150a984937a6ec0a3a8eb3ffafae8c 2024-12-06T05:39:58,933 INFO [StoreOpener-f711afceea9ea8582107efd7e8aa74eb-1 {}] regionserver.HStore(327): Store=f711afceea9ea8582107efd7e8aa74eb/col, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:39:58,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 91b02cf9f147172bb99e96b9a5ebc6c0, server=b543b351fe95,44975,1733463597882 because future has completed 2024-12-06T05:39:58,938 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/71150a984937a6ec0a3a8eb3ffafae8c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:39:58,939 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1114): Opened 71150a984937a6ec0a3a8eb3ffafae8c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61494217, jitterRate=-0.08366475999355316}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:39:58,939 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 71150a984937a6ec0a3a8eb3ffafae8c 2024-12-06T05:39:58,939 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1006): Region open journal for 71150a984937a6ec0a3a8eb3ffafae8c: Running coprocessor pre-open hook at 1733463598916Writing region info on filesystem at 1733463598916Initializing all the Stores at 1733463598918 (+2 ms)Instantiating store for column family {NAME => 'col', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463598918Cleaning up temporary data from old regions at 1733463598929 (+11 ms)Running coprocessor post-open hooks at 1733463598939 (+10 ms)Region opened successfully at 1733463598939 2024-12-06T05:39:58,940 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2236): Post open deploy tasks for testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1733463598004.71150a984937a6ec0a3a8eb3ffafae8c., pid=16, masterSystemTime=1733463598906 2024-12-06T05:39:58,935 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1038): replaying wal for f711afceea9ea8582107efd7e8aa74eb 2024-12-06T05:39:58,943 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/f711afceea9ea8582107efd7e8aa74eb 2024-12-06T05:39:58,944 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/f711afceea9ea8582107efd7e8aa74eb 2024-12-06T05:39:58,945 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1048): stopping wal replay for f711afceea9ea8582107efd7e8aa74eb 2024-12-06T05:39:58,945 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1060): Cleaning up temporary data for f711afceea9ea8582107efd7e8aa74eb 2024-12-06T05:39:58,946 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2266): Finished post open deploy task for testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1733463598004.71150a984937a6ec0a3a8eb3ffafae8c. 2024-12-06T05:39:58,946 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(153): Opened testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1733463598004.71150a984937a6ec0a3a8eb3ffafae8c. 2024-12-06T05:39:58,946 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1733463598004.59e2fa4009ebd132f92d2a7d7e292ba0. 2024-12-06T05:39:58,946 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 59e2fa4009ebd132f92d2a7d7e292ba0, NAME => 'testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1733463598004.59e2fa4009ebd132f92d2a7d7e292ba0.', STARTKEY => '1', ENDKEY => '2'} 2024-12-06T05:39:58,947 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testLogSplittingAfterMasterRecoveryDueToZKExpiry 59e2fa4009ebd132f92d2a7d7e292ba0 2024-12-06T05:39:58,947 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1733463598004.59e2fa4009ebd132f92d2a7d7e292ba0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:58,947 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 59e2fa4009ebd132f92d2a7d7e292ba0 2024-12-06T05:39:58,947 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 59e2fa4009ebd132f92d2a7d7e292ba0 2024-12-06T05:39:58,947 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=71150a984937a6ec0a3a8eb3ffafae8c, regionState=OPEN, openSeqNum=2, regionLocation=b543b351fe95,42121,1733463596109 2024-12-06T05:39:58,950 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=9 2024-12-06T05:39:58,951 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 91b02cf9f147172bb99e96b9a5ebc6c0, server=b543b351fe95,44975,1733463597882 in 222 msec 2024-12-06T05:39:58,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 71150a984937a6ec0a3a8eb3ffafae8c, server=b543b351fe95,42121,1733463596109 because future has completed 2024-12-06T05:39:58,957 INFO [StoreOpener-4e5ef5686145b3308e84acf2485afdef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family col of region 4e5ef5686145b3308e84acf2485afdef 2024-12-06T05:39:58,959 INFO [StoreOpener-4e5ef5686145b3308e84acf2485afdef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e5ef5686145b3308e84acf2485afdef columnFamilyName col 2024-12-06T05:39:58,959 DEBUG [StoreOpener-4e5ef5686145b3308e84acf2485afdef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:58,960 INFO [StoreOpener-4e5ef5686145b3308e84acf2485afdef-1 {}] regionserver.HStore(327): Store=4e5ef5686145b3308e84acf2485afdef/col, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:39:58,960 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 4e5ef5686145b3308e84acf2485afdef 2024-12-06T05:39:58,961 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/4e5ef5686145b3308e84acf2485afdef 2024-12-06T05:39:58,961 INFO [StoreOpener-59e2fa4009ebd132f92d2a7d7e292ba0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family col of region 59e2fa4009ebd132f92d2a7d7e292ba0 2024-12-06T05:39:58,966 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/4e5ef5686145b3308e84acf2485afdef 2024-12-06T05:39:58,967 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1093): writing seq id for f711afceea9ea8582107efd7e8aa74eb 2024-12-06T05:39:58,967 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 4e5ef5686145b3308e84acf2485afdef 2024-12-06T05:39:58,967 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 4e5ef5686145b3308e84acf2485afdef 2024-12-06T05:39:58,968 INFO [StoreOpener-59e2fa4009ebd132f92d2a7d7e292ba0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59e2fa4009ebd132f92d2a7d7e292ba0 columnFamilyName col 2024-12-06T05:39:58,968 DEBUG [StoreOpener-59e2fa4009ebd132f92d2a7d7e292ba0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:58,970 INFO [StoreOpener-59e2fa4009ebd132f92d2a7d7e292ba0-1 {}] regionserver.HStore(327): Store=59e2fa4009ebd132f92d2a7d7e292ba0/col, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:39:58,970 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 59e2fa4009ebd132f92d2a7d7e292ba0 2024-12-06T05:39:58,972 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/59e2fa4009ebd132f92d2a7d7e292ba0 2024-12-06T05:39:58,972 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 4e5ef5686145b3308e84acf2485afdef 2024-12-06T05:39:58,973 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/59e2fa4009ebd132f92d2a7d7e292ba0 2024-12-06T05:39:58,973 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 59e2fa4009ebd132f92d2a7d7e292ba0 2024-12-06T05:39:58,973 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 59e2fa4009ebd132f92d2a7d7e292ba0 2024-12-06T05:39:58,973 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=91b02cf9f147172bb99e96b9a5ebc6c0, ASSIGN in 402 msec 2024-12-06T05:39:58,974 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=8 2024-12-06T05:39:58,974 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 71150a984937a6ec0a3a8eb3ffafae8c, server=b543b351fe95,42121,1733463596109 in 212 msec 2024-12-06T05:39:58,978 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/4e5ef5686145b3308e84acf2485afdef/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:39:58,978 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=71150a984937a6ec0a3a8eb3ffafae8c, ASSIGN in 425 msec 2024-12-06T05:39:58,978 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 59e2fa4009ebd132f92d2a7d7e292ba0 2024-12-06T05:39:58,979 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 4e5ef5686145b3308e84acf2485afdef; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61351813, jitterRate=-0.08578674495220184}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:39:58,979 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4e5ef5686145b3308e84acf2485afdef 2024-12-06T05:39:58,979 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 4e5ef5686145b3308e84acf2485afdef: Running coprocessor pre-open hook at 1733463598933Writing region info on filesystem at 1733463598933Initializing all the Stores at 1733463598939 (+6 ms)Instantiating store for column family {NAME => 'col', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463598939Cleaning up temporary data from old regions at 1733463598967 (+28 ms)Running coprocessor post-open hooks at 1733463598979 (+12 ms)Region opened successfully at 1733463598979 2024-12-06T05:39:58,980 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., pid=12, masterSystemTime=1733463598875 2024-12-06T05:39:58,980 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/f711afceea9ea8582107efd7e8aa74eb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:39:58,981 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1114): Opened f711afceea9ea8582107efd7e8aa74eb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67892336, jitterRate=0.011674642562866211}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:39:58,981 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f711afceea9ea8582107efd7e8aa74eb 2024-12-06T05:39:58,981 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1006): Region open journal for f711afceea9ea8582107efd7e8aa74eb: Running coprocessor pre-open hook at 1733463598912Writing region info on filesystem at 1733463598912Initializing all the Stores at 1733463598917 (+5 ms)Instantiating store for column family {NAME => 'col', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463598918 (+1 ms)Cleaning up temporary data from old regions at 1733463598945 (+27 ms)Running coprocessor post-open hooks at 1733463598981 (+36 ms)Region opened successfully at 1733463598981 2024-12-06T05:39:58,982 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegionServer(2236): Post open deploy tasks for testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1733463598004.f711afceea9ea8582107efd7e8aa74eb., pid=14, masterSystemTime=1733463598901 2024-12-06T05:39:58,983 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef. 2024-12-06T05:39:58,983 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef. 2024-12-06T05:39:58,985 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4e5ef5686145b3308e84acf2485afdef, regionState=OPEN, openSeqNum=2, regionLocation=b543b351fe95,44975,1733463597882 2024-12-06T05:39:58,987 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegionServer(2266): Finished post open deploy task for testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1733463598004.f711afceea9ea8582107efd7e8aa74eb. 2024-12-06T05:39:58,987 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] handler.AssignRegionHandler(153): Opened testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1733463598004.f711afceea9ea8582107efd7e8aa74eb. 2024-12-06T05:39:58,987 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1733463598004.ad0165368b2f0ee58b25d3d51e5286d5. 2024-12-06T05:39:58,987 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => ad0165368b2f0ee58b25d3d51e5286d5, NAME => 'testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1733463598004.ad0165368b2f0ee58b25d3d51e5286d5.', STARTKEY => '2', ENDKEY => '3'} 2024-12-06T05:39:58,987 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=f711afceea9ea8582107efd7e8aa74eb, regionState=OPEN, openSeqNum=2, regionLocation=b543b351fe95,43157,1733463596184 2024-12-06T05:39:58,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e5ef5686145b3308e84acf2485afdef, server=b543b351fe95,44975,1733463597882 because future has completed 2024-12-06T05:39:58,988 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testLogSplittingAfterMasterRecoveryDueToZKExpiry ad0165368b2f0ee58b25d3d51e5286d5 2024-12-06T05:39:58,988 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1733463598004.ad0165368b2f0ee58b25d3d51e5286d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:58,988 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for ad0165368b2f0ee58b25d3d51e5286d5 2024-12-06T05:39:58,988 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for ad0165368b2f0ee58b25d3d51e5286d5 2024-12-06T05:39:58,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure f711afceea9ea8582107efd7e8aa74eb, server=b543b351fe95,43157,1733463596184 because future has completed 2024-12-06T05:39:58,994 INFO [StoreOpener-ad0165368b2f0ee58b25d3d51e5286d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family col of region ad0165368b2f0ee58b25d3d51e5286d5 2024-12-06T05:39:58,997 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=5 2024-12-06T05:39:58,997 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/59e2fa4009ebd132f92d2a7d7e292ba0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:39:58,998 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 4e5ef5686145b3308e84acf2485afdef, server=b543b351fe95,44975,1733463597882 in 269 msec 2024-12-06T05:39:58,998 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 59e2fa4009ebd132f92d2a7d7e292ba0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75151587, jitterRate=0.119845911860466}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:39:58,998 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 59e2fa4009ebd132f92d2a7d7e292ba0 2024-12-06T05:39:58,998 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 59e2fa4009ebd132f92d2a7d7e292ba0: Running coprocessor pre-open hook at 1733463598947Writing region info on filesystem at 1733463598947Initializing all the Stores at 1733463598951 (+4 ms)Instantiating store for column family {NAME => 'col', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463598951Cleaning up temporary data from old regions at 1733463598973 (+22 ms)Running coprocessor post-open hooks at 1733463598998 (+25 ms)Region opened successfully at 1733463598998 2024-12-06T05:39:59,000 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1733463598004.59e2fa4009ebd132f92d2a7d7e292ba0., pid=15, masterSystemTime=1733463598906 2024-12-06T05:39:59,000 INFO [StoreOpener-ad0165368b2f0ee58b25d3d51e5286d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ad0165368b2f0ee58b25d3d51e5286d5 columnFamilyName col 2024-12-06T05:39:59,000 DEBUG [StoreOpener-ad0165368b2f0ee58b25d3d51e5286d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:59,001 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=10 2024-12-06T05:39:59,001 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure f711afceea9ea8582107efd7e8aa74eb, server=b543b351fe95,43157,1733463596184 in 250 msec 2024-12-06T05:39:59,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=4e5ef5686145b3308e84acf2485afdef, ASSIGN in 449 msec 2024-12-06T05:39:59,003 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1733463598004.59e2fa4009ebd132f92d2a7d7e292ba0. 2024-12-06T05:39:59,004 INFO [StoreOpener-ad0165368b2f0ee58b25d3d51e5286d5-1 {}] regionserver.HStore(327): Store=ad0165368b2f0ee58b25d3d51e5286d5/col, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:39:59,004 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1733463598004.59e2fa4009ebd132f92d2a7d7e292ba0. 2024-12-06T05:39:59,004 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for ad0165368b2f0ee58b25d3d51e5286d5 2024-12-06T05:39:59,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=f711afceea9ea8582107efd7e8aa74eb, ASSIGN in 452 msec 2024-12-06T05:39:59,005 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/ad0165368b2f0ee58b25d3d51e5286d5 2024-12-06T05:39:59,005 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=59e2fa4009ebd132f92d2a7d7e292ba0, regionState=OPEN, openSeqNum=2, regionLocation=b543b351fe95,42121,1733463596109 2024-12-06T05:39:59,005 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/ad0165368b2f0ee58b25d3d51e5286d5 2024-12-06T05:39:59,006 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for ad0165368b2f0ee58b25d3d51e5286d5 2024-12-06T05:39:59,006 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for ad0165368b2f0ee58b25d3d51e5286d5 2024-12-06T05:39:59,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59e2fa4009ebd132f92d2a7d7e292ba0, server=b543b351fe95,42121,1733463596109 because future has completed 2024-12-06T05:39:59,009 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for ad0165368b2f0ee58b25d3d51e5286d5 2024-12-06T05:39:59,012 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/ad0165368b2f0ee58b25d3d51e5286d5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:39:59,013 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened ad0165368b2f0ee58b25d3d51e5286d5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62136769, jitterRate=-0.07408998906612396}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:39:59,013 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ad0165368b2f0ee58b25d3d51e5286d5 2024-12-06T05:39:59,013 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=6 2024-12-06T05:39:59,013 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 59e2fa4009ebd132f92d2a7d7e292ba0, server=b543b351fe95,42121,1733463596109 in 259 msec 2024-12-06T05:39:59,014 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for ad0165368b2f0ee58b25d3d51e5286d5: Running coprocessor pre-open hook at 1733463598988Writing region info on filesystem at 1733463598988Initializing all the Stores at 1733463598990 (+2 ms)Instantiating store for column family {NAME => 'col', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463598990Cleaning up temporary data from old regions at 1733463599006 (+16 ms)Running coprocessor post-open hooks at 1733463599013 (+7 ms)Region opened successfully at 1733463599014 (+1 ms) 2024-12-06T05:39:59,015 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1733463598004.ad0165368b2f0ee58b25d3d51e5286d5., pid=13, masterSystemTime=1733463598901 2024-12-06T05:39:59,016 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=59e2fa4009ebd132f92d2a7d7e292ba0, ASSIGN in 464 msec 2024-12-06T05:39:59,018 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1733463598004.ad0165368b2f0ee58b25d3d51e5286d5. 2024-12-06T05:39:59,018 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1733463598004.ad0165368b2f0ee58b25d3d51e5286d5. 2024-12-06T05:39:59,019 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ad0165368b2f0ee58b25d3d51e5286d5, regionState=OPEN, openSeqNum=2, regionLocation=b543b351fe95,43157,1733463596184 2024-12-06T05:39:59,022 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure ad0165368b2f0ee58b25d3d51e5286d5, server=b543b351fe95,43157,1733463596184 because future has completed 2024-12-06T05:39:59,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=7 2024-12-06T05:39:59,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure ad0165368b2f0ee58b25d3d51e5286d5, server=b543b351fe95,43157,1733463596184 in 294 msec 2024-12-06T05:39:59,030 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=7, resume processing ppid=4 2024-12-06T05:39:59,030 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=ad0165368b2f0ee58b25d3d51e5286d5, ASSIGN in 477 msec 2024-12-06T05:39:59,031 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T05:39:59,031 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testLogSplittingAfterMasterRecoveryDueToZKExpiry","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733463599031"}]},"ts":"1733463599031"} 2024-12-06T05:39:59,034 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testLogSplittingAfterMasterRecoveryDueToZKExpiry, state=ENABLED in hbase:meta 2024-12-06T05:39:59,035 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T05:39:59,040 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry in 1.0300 sec 2024-12-06T05:39:59,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T05:39:59,150 INFO [RPCClient-NioEventLoopGroup-6-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testLogSplittingAfterMasterRecoveryDueToZKExpiry completed 2024-12-06T05:39:59,151 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T05:39:59,156 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testLogSplittingAfterMasterRecoveryDueToZKExpiry', row='\x00\x00\x00\x00', locateType=CURRENT is [region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., hostname=b543b351fe95,44975,1733463597882, seqNum=2] 2024-12-06T05:39:59,156 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T05:39:59,158 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-19-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43844, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T05:39:59,174 ERROR [Time-limited test {}] master.HMaster(3218): Master server abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-06T05:39:59,174 ERROR [Time-limited test {}] master.HMaster(3223): ***** ABORTING master b543b351fe95,42019,1733463595907: Test recovery from zk session expired ***** org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired at org.apache.hadoop.hbase.TestZooKeeper.testLogSplittingAfterMasterRecoveryDueToZKExpiry(TestZooKeeper.java:269) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:39:59,174 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master 'b543b351fe95,42019,1733463595907' ***** 2024-12-06T05:39:59,174 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: Stopped by Time-limited test 2024-12-06T05:39:59,174 INFO [M:0;b543b351fe95:42019 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T05:39:59,174 INFO [M:0;b543b351fe95:42019 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;b543b351fe95:42019. 2024-12-06T05:39:59,175 DEBUG [M:0;b543b351fe95:42019 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T05:39:59,175 DEBUG [M:0;b543b351fe95:42019 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:39:59,175 INFO [M:0;b543b351fe95:42019 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T05:39:59,175 DEBUG [M:0;b543b351fe95:42019 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-06T05:39:59,175 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-06T05:39:59,175 DEBUG [M:0;b543b351fe95:42019 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-06T05:39:59,175 DEBUG [master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.large.0-1733463596924 {}] cleaner.HFileCleaner(306): Exit Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.large.0-1733463596924,5,FailOnTimeoutGroup] 2024-12-06T05:39:59,175 INFO [M:0;b543b351fe95:42019 {}] hbase.ChoreService(370): Chore service for: master/b543b351fe95:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-06T05:39:59,175 DEBUG [master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.small.0-1733463596924 {}] cleaner.HFileCleaner(306): Exit Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.small.0-1733463596924,5,FailOnTimeoutGroup] 2024-12-06T05:39:59,175 INFO [M:0;b543b351fe95:42019 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T05:39:59,175 DEBUG [M:0;b543b351fe95:42019 {}] master.HMaster(1795): Stopping service threads 2024-12-06T05:39:59,175 INFO [M:0;b543b351fe95:42019 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-06T05:39:59,176 INFO [M:0;b543b351fe95:42019 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-06T05:39:59,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:59,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42121-0x100ae0a5c730014, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:59,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:59,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42121-0x100ae0a5c730014, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:59,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:59,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43157-0x100ae0a5c730015, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:59,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43157-0x100ae0a5c730015, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:59,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:59,177 ERROR [M:0;b543b351fe95:42019 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] 2024-12-06T05:39:59,177 INFO [M:0;b543b351fe95:42019 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=true 2024-12-06T05:39:59,177 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-06T05:39:59,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44975-0x100ae0a5c730017, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:59,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44975-0x100ae0a5c730017, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:59,178 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:39:59,178 DEBUG [zk-event-processor-pool-0 {}] master.ActiveMasterManager(204): No master available. Notifying waiting threads 2024-12-06T05:39:59,178 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43157-0x100ae0a5c730015, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:39:59,179 DEBUG [M:0;b543b351fe95:42019 {}] zookeeper.ZKUtil(347): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-06T05:39:59,179 WARN [M:0;b543b351fe95:42019 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-06T05:39:59,179 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42121-0x100ae0a5c730014, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:39:59,179 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44975-0x100ae0a5c730017, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:39:59,179 INFO [M:0;b543b351fe95:42019 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/.lastflushedseqids 2024-12-06T05:39:59,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:59,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42121-0x100ae0a5c730014, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:59,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44975-0x100ae0a5c730017, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:59,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:59,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:59,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43157-0x100ae0a5c730015, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T05:39:59,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43157-0x100ae0a5c730015, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:39:59,182 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T05:39:59,182 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T05:39:59,182 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testLogSplittingAfterMasterRecoveryDueToZKExpiry,, stopping at row=testLogSplittingAfterMasterRecoveryDueToZKExpiry ,, for max=2147483647 with caching=100 2024-12-06T05:39:59,182 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b543b351fe95,42551,1733463595943 from backup master directory 2024-12-06T05:39:59,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44975-0x100ae0a5c730017, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:59,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43157-0x100ae0a5c730015, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:59,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b543b351fe95,42551,1733463595943 2024-12-06T05:39:59,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:59,183 WARN [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T05:39:59,183 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b543b351fe95,42551,1733463595943 2024-12-06T05:39:59,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:59,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42121-0x100ae0a5c730014, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:39:59,190 DEBUG [HMaster-EventLoopGroup-15-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b543b351fe95,42019,1733463595907 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:39:59,193 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 6 regions for table testLogSplittingAfterMasterRecoveryDueToZKExpiry 2024-12-06T05:39:59,194 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef. 2024-12-06T05:39:59,194 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing b543b351fe95,44975,1733463597882 2024-12-06T05:39:59,194 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server b543b351fe95,44975,1733463597882: Simulated kill ***** 2024-12-06T05:39:59,194 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-06T05:39:59,194 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-06T05:39:59,194 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-06T05:39:59,195 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-06T05:39:59,195 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-06T05:39:59,197 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 369607072 }, "NonHeapMemoryUsage": { "committed": 163905536, "init": 7667712, "max": -1, "used": 161424056 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 14574, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 57, "ProcessCallTime_min": 0, "ProcessCallTime_max": 7, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 2, "ProcessCallTime_75th_percentile": 3, "ProcessCallTime_90th_percentile": 4, "ProcessCallTime_95th_percentile": 6, "ProcessCallTime_98th_percentile": 7, "ProcessCallTime_99th_percentile": 7, "ProcessCallTime_99.9th_percentile": 7, "ProcessCallTime_TimeRangeCount_0-1": 11, "ProcessCallTime_TimeRangeCount_1-3": 12, "ProcessCallTime_TimeRangeCount_3-10": 5, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 57, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 1, "QueueCallTime_95th_percentile": 1, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 28, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 57, "TotalCallTime_min": 0, "TotalCallTime_max": 7, "TotalCallTime_mean": 2, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 3, "TotalCallTime_90th_percentile": 4, "TotalCallTime_95th_percentile": 6, "TotalCallTime_98th_percentile": 7, "TotalCallTime_99th_percentile": 7, "TotalCallTime_99.9th_percentile": 7, "TotalCallTime_TimeRangeCount_0-1": 10, "TotalCallTime_TimeRangeCount_1-3": 13, "TotalCallTime_TimeRangeCount_3-10": 5, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 57, "ResponseSize_min": 0, "ResponseSize_max": 1179, "ResponseSize_mean": 104, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 23, "ResponseSize_90th_percentile": 175, "ResponseSize_95th_percentile": 532, "ResponseSize_98th_percentile": 920, "ResponseSize_99th_percentile": 1049, "ResponseSize_99.9th_percentile": 1166, "ResponseSize_SizeRangeCount_0-10": 19, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 5, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 57, "RequestSize_min": 89, "RequestSize_max": 402, "RequestSize_mean": 175, "RequestSize_25th_percentile": 110, "RequestSize_median": 125, "RequestSize_75th_percentile": 225, "RequestSize_90th_percentile": 342, "RequestSize_95th_percentile": 360, "RequestSize_98th_percentile": 385, "RequestSize_99th_percentile": 393, "RequestSize_99.9th_percentile": 401, "RequestSize_SizeRangeCount_10-100": 5, "RequestSize_SizeRangeCount_100-1000": 23, "sentBytes": 4543 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:58086", "tag.serverName": "b543b351fe95,37775,1733463581162", "tag.clusterId": "f24feb01-2567-4d5e-a4c6-86f800bb290a", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95", "regionCount": 0, "storeCount": 0, "hlogFileCount": 4, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733463581162, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 77674, "localBytesRead": 77674, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 30370, "ByteBuffAllocatorPoolAllocationBytes": 399360, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 18, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 8, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 8, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 8, "ScanTime_min": 1, "ScanTime_max": 1, "ScanTime_mean": 1, "ScanTime_25th_percentile": 1, "ScanTime_median": 1, "ScanTime_75th_percentile": 1, "ScanTime_90th_percentile": 1, "ScanTime_95th_percentile": 1, "ScanTime_98th_percentile": 1, "ScanTime_99th_percentile": 1, "ScanTime_99.9th_percentile": 1, "ScanTime_TimeRangeCount_0-1": 1, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 8, "Put_min": 2, "Put_max": 3, "Put_mean": 2, "Put_25th_percentile": 2, "Put_median": 2, "Put_75th_percentile": 3, "Put_90th_percentile": 3, "Put_95th_percentile": 3, "Put_98th_percentile": 3, "Put_99th_percentile": 3, "Put_99.9th_percentile": 3, "Put_TimeRangeCount_1-3": 6, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 8, "ScanSize_min": 1072, "ScanSize_max": 1072, "ScanSize_mean": 1072, "ScanSize_25th_percentile": 1072, "ScanSize_median": 1072, "ScanSize_75th_percentile": 1072, "ScanSize_90th_percentile": 1072, "ScanSize_95th_percentile": 1072, "ScanSize_98th_percentile": 1072, "ScanSize_99th_percentile": 1072, "ScanSize_99.9th_percentile": 1072, "ScanSize_SizeRangeCount_100-1000": 1, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-06T05:39:59,198 DEBUG [HMaster-EventLoopGroup-15-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b543b351fe95,42019,1733463595907 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:39:59,199 WARN [Time-limited test {}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.regionserver.RegionServerAbortedException): Server b543b351fe95,42019,1733463595907 aborting at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:?] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:?] at org.apache.hadoop.hbase.Abortable.abort(Abortable.java:45) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.kill(HRegionServer.java:2485) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.kill(SingleProcessHBaseCluster.java:177) ~[test-classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster.killRegionServer(SingleProcessHBaseCluster.java:268) ~[test-classes/:?] at org.apache.hadoop.hbase.TestZooKeeper.testLogSplittingAfterMasterRecoveryDueToZKExpiry(TestZooKeeper.java:272) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: Server b543b351fe95,42019,1733463595907 aborting at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-12-06T05:39:59,203 INFO [master/b543b351fe95:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:39:59,201 DEBUG [HMaster-EventLoopGroup-15-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b543b351fe95,42019,1733463595907 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:39:59,204 INFO [master/b543b351fe95:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-06T05:39:59,205 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b543b351fe95,44975,1733463597882' ***** 2024-12-06T05:39:59,205 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-12-06T05:39:59,205 INFO [RS:2;b543b351fe95:44975 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T05:39:59,205 INFO [RS:2;b543b351fe95:44975 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-06T05:39:59,205 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T05:39:59,205 INFO [RS:2;b543b351fe95:44975 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-06T05:39:59,205 INFO [RS:2;b543b351fe95:44975 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T05:39:59,205 INFO [RS:2;b543b351fe95:44975 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;b543b351fe95:44975. 2024-12-06T05:39:59,205 DEBUG [RS:2;b543b351fe95:44975 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T05:39:59,205 DEBUG [RS:2;b543b351fe95:44975 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:39:59,206 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-19-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b543b351fe95,44975,1733463597882 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:39:59,208 DEBUG [RPCClient-NioEventLoopGroup-6-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., hostname=b543b351fe95,44975,1733463597882, seqNum=2 , the old value is region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., hostname=b543b351fe95,44975,1733463597882, seqNum=2, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b543b351fe95,44975,1733463597882 aborting 2024-12-06T05:39:59,208 DEBUG [RPCClient-NioEventLoopGroup-6-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., hostname=b543b351fe95,44975,1733463597882, seqNum=2 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b543b351fe95,44975,1733463597882 aborting 2024-12-06T05:39:59,208 DEBUG [RPCClient-NioEventLoopGroup-6-9 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., hostname=b543b351fe95,44975,1733463597882, seqNum=2 from cache 2024-12-06T05:39:59,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741897_1074 (size=1682) 2024-12-06T05:39:59,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741897_1074 (size=1682) 2024-12-06T05:39:59,210 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,44975,1733463597882/b543b351fe95%2C44975%2C1733463597882.1733463598114 not finished, retry = 0 2024-12-06T05:39:59,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741898_1075 (size=463) 2024-12-06T05:39:59,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741898_1075 (size=463) 2024-12-06T05:39:59,222 INFO [master/b543b351fe95:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=2) cost 18ms. 2024-12-06T05:39:59,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741899_1076 (size=196) 2024-12-06T05:39:59,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741899_1076 (size=196) 2024-12-06T05:39:59,267 INFO [regionserver/b543b351fe95:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T05:39:59,294 DEBUG [RS:0;b543b351fe95:42121 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T05:39:59,296 INFO [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37265, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T05:39:59,296 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42551 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:59,307 DEBUG [RS:1;b543b351fe95:43157 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T05:39:59,309 INFO [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33851, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T05:39:59,310 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42551 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:59,311 DEBUG [RS:2;b543b351fe95:44975 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:39:59,311 INFO [RS:2;b543b351fe95:44975 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T05:39:59,311 INFO [RS:2;b543b351fe95:44975 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44975 2024-12-06T05:39:59,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T05:39:59,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T05:39:59,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44975-0x100ae0a5c730017, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b543b351fe95,44975,1733463597882 2024-12-06T05:39:59,314 INFO [RS:2;b543b351fe95:44975 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T05:39:59,322 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testLogSplittingAfterMasterRecoveryDueToZKExpiry', row='', locateType=CURRENT is [region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., hostname=b543b351fe95,44975,1733463597882, seqNum=2] 2024-12-06T05:39:59,324 WARN [RPCClient-NioEventLoopGroup-6-9 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server b543b351fe95:44975 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: b543b351fe95/172.17.0.2:44975 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:39:59,324 DEBUG [RPCClient-NioEventLoopGroup-6-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., hostname=b543b351fe95,44975,1733463597882, seqNum=2 , the old value is region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., hostname=b543b351fe95,44975,1733463597882, seqNum=2, error=java.net.ConnectException: Call to address=b543b351fe95:44975 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: b543b351fe95/172.17.0.2:44975 2024-12-06T05:39:59,324 DEBUG [RPCClient-NioEventLoopGroup-6-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., hostname=b543b351fe95,44975,1733463597882, seqNum=2 is java.net.ConnectException: Connection refused 2024-12-06T05:39:59,324 DEBUG [RPCClient-NioEventLoopGroup-6-9 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., hostname=b543b351fe95,44975,1733463597882, seqNum=2 from cache 2024-12-06T05:39:59,324 DEBUG [RPCClient-NioEventLoopGroup-6-9 {}] ipc.FailedServers(52): Added failed server with address b543b351fe95:44975 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: b543b351fe95/172.17.0.2:44975 2024-12-06T05:39:59,399 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42551 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:59,413 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42551 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:59,417 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44975-0x100ae0a5c730017, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:39:59,417 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44975-0x100ae0a5c730017, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:39:59,417 INFO [RS:2;b543b351fe95:44975 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T05:39:59,417 INFO [RS:2;b543b351fe95:44975 {}] regionserver.HRegionServer(1031): Exiting; stopping=b543b351fe95,44975,1733463597882; zookeeper connection closed. 2024-12-06T05:39:59,417 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6f6db211 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6f6db211 2024-12-06T05:39:59,502 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42551 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:59,515 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42551 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:59,532 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testLogSplittingAfterMasterRecoveryDueToZKExpiry', row='', locateType=CURRENT is [region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., hostname=b543b351fe95,44975,1733463597882, seqNum=2] 2024-12-06T05:39:59,532 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] ipc.AbstractRpcClient(357): Not trying to connect to b543b351fe95:44975 this server is in the failed servers list 2024-12-06T05:39:59,533 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., hostname=b543b351fe95,44975,1733463597882, seqNum=2 , the old value is region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., hostname=b543b351fe95,44975,1733463597882, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=b543b351fe95:44975 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: b543b351fe95:44975 2024-12-06T05:39:59,533 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., hostname=b543b351fe95,44975,1733463597882, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: b543b351fe95:44975 2024-12-06T05:39:59,533 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., hostname=b543b351fe95,44975,1733463597882, seqNum=2 from cache 2024-12-06T05:39:59,605 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42551 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:59,610 INFO [M:0;b543b351fe95:42019 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-06T05:39:59,610 INFO [M:0;b543b351fe95:42019 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=true 2024-12-06T05:39:59,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741884_1061 (size=72145) 2024-12-06T05:39:59,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741884_1061 (size=72145) 2024-12-06T05:39:59,618 DEBUG [M:0;b543b351fe95:42019 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T05:39:59,618 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42551 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:59,618 INFO [M:0;b543b351fe95:42019 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:59,618 DEBUG [M:0;b543b351fe95:42019 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:59,618 DEBUG [M:0;b543b351fe95:42019 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T05:39:59,618 DEBUG [M:0;b543b351fe95:42019 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:59,619 ERROR [M:0;b543b351fe95:42019 {}] regionserver.HRegion(1960): Memstore data size is 63656 in region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:59,619 INFO [M:0;b543b351fe95:42019 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:39:59,619 DEBUG [M:0;b543b351fe95:42019 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733463599618Disabling compacts and flushes for region at 1733463599618Disabling writes for close at 1733463599618Writing region close event to WAL at 1733463599618Closed at 1733463599619 (+1 ms) 2024-12-06T05:39:59,619 INFO [M:0;b543b351fe95:42019 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-06T05:39:59,619 INFO [M:0;b543b351fe95:42019 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42019 2024-12-06T05:39:59,619 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T05:39:59,619 INFO [M:0;b543b351fe95:42019 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T05:39:59,654 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T05:39:59,655 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-06T05:39:59,655 INFO [master/b543b351fe95:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:39:59,664 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(342): old store file tracker DEFAULT is the same with new store file tracker, skip migration 2024-12-06T05:39:59,667 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(316): Renamed hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,42019,1733463595907 to hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,42019,1733463595907-dead as it is dead 2024-12-06T05:39:59,668 INFO [master/b543b351fe95:0:becomeActiveMaster {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,42019,1733463595907-dead/b543b351fe95%2C42019%2C1733463595907.1733463596560 2024-12-06T05:39:59,668 INFO [master/b543b351fe95:0:becomeActiveMaster {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,42019,1733463595907-dead/b543b351fe95%2C42019%2C1733463595907.1733463596560 after 0ms 2024-12-06T05:39:59,669 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(328): Renamed hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,42019,1733463595907-dead/b543b351fe95%2C42019%2C1733463595907.1733463596560 to hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.wals/b543b351fe95%2C42019%2C1733463595907.1733463596560 2024-12-06T05:39:59,669 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(330): Delete empty local region wal dir hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,42019,1733463595907-dead 2024-12-06T05:39:59,669 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,42551,1733463595943 2024-12-06T05:39:59,670 INFO [master/b543b351fe95:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:39:59,672 INFO [master/b543b351fe95:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b543b351fe95%2C42551%2C1733463595943, suffix=, logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,42551,1733463595943, archiveDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/oldWALs, maxLogs=10 2024-12-06T05:39:59,685 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,42551,1733463595943/b543b351fe95%2C42551%2C1733463595943.1733463599672, exclude list is [], retry=0 2024-12-06T05:39:59,689 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-19-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-80ddb16c-ec66-4e39-a44c-0a80fa143251,DISK] 2024-12-06T05:39:59,689 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-19-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39561,DS-4485eaa6-367b-4084-97e1-b834fb8d8596,DISK] 2024-12-06T05:39:59,691 INFO [master/b543b351fe95:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,42551,1733463595943/b543b351fe95%2C42551%2C1733463595943.1733463599672 2024-12-06T05:39:59,692 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43559:43559),(127.0.0.1/127.0.0.1:44855:44855)] 2024-12-06T05:39:59,692 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:39:59,692 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:39:59,692 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:59,692 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:59,694 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:59,696 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-06T05:39:59,696 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:59,697 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:39:59,697 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:59,698 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-06T05:39:59,698 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:59,698 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:39:59,698 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:59,699 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-06T05:39:59,699 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:59,704 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:39:59,704 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:59,705 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-06T05:39:59,705 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:39:59,706 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:39:59,706 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:59,706 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(5516): Found 1 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.wals 2024-12-06T05:39:59,707 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.wals/b543b351fe95%2C42019%2C1733463595907.1733463596560 2024-12-06T05:39:59,707 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42551 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:59,721 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42551 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:59,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:39:59,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42019-0x100ae0a5c730012, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:39:59,721 INFO [M:0;b543b351fe95:42019 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T05:39:59,722 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(5793): Applied 163, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=131, path=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.wals/b543b351fe95%2C42019%2C1733463595907.1733463596560 2024-12-06T05:39:59,722 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=62.16 KB heapSize=75.20 KB 2024-12-06T05:39:59,737 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d55188621ec54721a1189c9be77e4e85 is 82, key is hbase:meta,,1/info:regioninfo/1733463597703/Put/seqid=0 2024-12-06T05:39:59,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741901_1078 (size=5672) 2024-12-06T05:39:59,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741901_1078 (size=5672) 2024-12-06T05:39:59,743 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d55188621ec54721a1189c9be77e4e85 2024-12-06T05:39:59,770 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ee2d9c43b9c94cbe99898787058fcf56 is 1200, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733463599036/Put/seqid=0 2024-12-06T05:39:59,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741902_1079 (size=68131) 2024-12-06T05:39:59,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741902_1079 (size=68131) 2024-12-06T05:39:59,778 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=61.48 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ee2d9c43b9c94cbe99898787058fcf56 2024-12-06T05:39:59,809 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e4b996c813444291b780344207799bfd is 69, key is b543b351fe95,42121,1733463596109/rs:state/1733463597010/Put/seqid=0 2024-12-06T05:39:59,809 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42551 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:59,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741903_1080 (size=5294) 2024-12-06T05:39:59,820 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=195 B at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e4b996c813444291b780344207799bfd 2024-12-06T05:39:59,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741903_1080 (size=5294) 2024-12-06T05:39:59,824 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42551 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:59,830 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d55188621ec54721a1189c9be77e4e85 as hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d55188621ec54721a1189c9be77e4e85 2024-12-06T05:39:59,842 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testLogSplittingAfterMasterRecoveryDueToZKExpiry', row='', locateType=CURRENT is [region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., hostname=b543b351fe95,44975,1733463597882, seqNum=2] 2024-12-06T05:39:59,843 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d55188621ec54721a1189c9be77e4e85, entries=8, sequenceid=131, filesize=5.5 K 2024-12-06T05:39:59,843 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] ipc.AbstractRpcClient(357): Not trying to connect to b543b351fe95:44975 this server is in the failed servers list 2024-12-06T05:39:59,843 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., hostname=b543b351fe95,44975,1733463597882, seqNum=2 , the old value is region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., hostname=b543b351fe95,44975,1733463597882, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=b543b351fe95:44975 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: b543b351fe95:44975 2024-12-06T05:39:59,843 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., hostname=b543b351fe95,44975,1733463597882, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: b543b351fe95:44975 2024-12-06T05:39:59,843 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., hostname=b543b351fe95,44975,1733463597882, seqNum=2 from cache 2024-12-06T05:39:59,844 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ee2d9c43b9c94cbe99898787058fcf56 as hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ee2d9c43b9c94cbe99898787058fcf56 2024-12-06T05:39:59,853 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ee2d9c43b9c94cbe99898787058fcf56, entries=152, sequenceid=131, filesize=66.5 K 2024-12-06T05:39:59,854 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e4b996c813444291b780344207799bfd as hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e4b996c813444291b780344207799bfd 2024-12-06T05:39:59,861 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e4b996c813444291b780344207799bfd, entries=3, sequenceid=131, filesize=5.2 K 2024-12-06T05:39:59,862 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(3140): Finished flush of dataSize ~62.16 KB/63656, heapSize ~74.91 KB/76704, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 140ms, sequenceid=131, compaction requested=false; wal=null 2024-12-06T05:39:59,863 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.wals/b543b351fe95%2C42019%2C1733463595907.1733463596560 2024-12-06T05:39:59,864 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:59,864 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:59,865 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T05:39:59,866 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:39:59,869 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/131.seqid, newMaxSeqId=131, maxSeqId=1 2024-12-06T05:39:59,870 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=132; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67924104, jitterRate=0.012148022651672363}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T05:39:59,871 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733463599692Initializing all the Stores at 1733463599693 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463599693Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463599694 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463599694Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463599694Obtaining lock to block concurrent updates at 1733463599722 (+28 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733463599722Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=63656, getHeapSize=76944, getOffHeapSize=0, getCellsCount=163 at 1733463599723 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733463599723Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733463599723Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733463599736 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733463599736Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733463599751 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733463599769 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733463599769Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733463599790 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733463599808 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733463599809 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5026ea96: reopening flushed file at 1733463599829 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@727f8d89: reopening flushed file at 1733463599843 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60213541: reopening flushed file at 1733463599853 (+10 ms)Finished flush of dataSize ~62.16 KB/63656, heapSize ~74.91 KB/76704, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 140ms, sequenceid=131, compaction requested=false; wal=null at 1733463599862 (+9 ms)Cleaning up temporary data from old regions at 1733463599864 (+2 ms)Region opened successfully at 1733463599871 (+7 ms) 2024-12-06T05:39:59,871 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-06T05:39:59,875 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22fbb777, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b543b351fe95/172.17.0.2:0 2024-12-06T05:39:59,878 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(884): The info family in master local region already has data in it, skip migrating... 2024-12-06T05:39:59,878 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-06T05:39:59,879 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-06T05:39:59,879 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-06T05:39:59,879 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-06T05:39:59,888 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-06T05:39:59,889 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry 2024-12-06T05:39:59,889 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-12-06T05:39:59,889 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-06T05:39:59,894 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b543b351fe95,42121,1733463596109, regionLocation=b543b351fe95,42121,1733463596109, openSeqNum=2 2024-12-06T05:39:59,894 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b543b351fe95,42121,1733463596109, table=hbase:meta, region=1588230740 2024-12-06T05:39:59,895 INFO [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b543b351fe95,42121,1733463596109, state=OPEN 2024-12-06T05:39:59,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:39:59,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42121-0x100ae0a5c730014, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:39:59,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43157-0x100ae0a5c730015, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:39:59,896 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:59,896 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:59,897 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:39:59,900 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 3 possibly 'live' servers, and 0 'splitting'. 2024-12-06T05:39:59,900 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ServerManager(517): Registering regionserver=b543b351fe95,43157,1733463596184 2024-12-06T05:39:59,901 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ServerManager(517): Registering regionserver=b543b351fe95,42121,1733463596109 2024-12-06T05:39:59,902 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b543b351fe95,44975,1733463597882 already deleted, retry=false 2024-12-06T05:39:59,902 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ServerManager(695): Processing expiration of b543b351fe95,44975,1733463597882 on b543b351fe95,42551,1733463595943 2024-12-06T05:39:59,903 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure b543b351fe95,44975,1733463597882, splitWal=true, meta=false 2024-12-06T05:39:59,903 INFO [master/b543b351fe95:0:becomeActiveMaster {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=17 for b543b351fe95,44975,1733463597882 (carryingMeta=false) b543b351fe95,44975,1733463597882/CRASHED/regionCount=0/lock=java.util.concurrent.locks.ReentrantReadWriteLock@1551cbde[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-06T05:39:59,905 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-06T05:39:59,906 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-06T05:39:59,906 INFO [master/b543b351fe95:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-06T05:39:59,907 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-06T05:39:59,910 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-06T05:39:59,911 INFO [master/b543b351fe95:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-06T05:39:59,912 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-06T05:39:59,912 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42551 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:59,914 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-06T05:39:59,915 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-06T05:39:59,916 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-06T05:39:59,919 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-06T05:39:59,919 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-06T05:39:59,920 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=b543b351fe95,42551,1733463595943, sessionid=0x100ae0a5c730013, setting cluster-up flag (Was=true) 2024-12-06T05:39:59,921 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-06T05:39:59,922 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b543b351fe95,42551,1733463595943 2024-12-06T05:39:59,924 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-06T05:39:59,925 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b543b351fe95,42551,1733463595943 2024-12-06T05:39:59,927 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ServerManager(1190): begin to load .lastflushedseqids at hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/.lastflushedseqids 2024-12-06T05:39:59,928 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42551 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T05:39:59,929 INFO [master/b543b351fe95:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-06T05:39:59,930 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b543b351fe95,42551,1733463595943 Number of backup masters: 0 Number of live region servers: 2 b543b351fe95,42121,1733463596109 b543b351fe95,43157,1733463596184 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-06T05:39:59,931 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:39:59,931 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:39:59,931 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:39:59,931 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:39:59,931 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b543b351fe95:0, corePoolSize=10, maxPoolSize=10 2024-12-06T05:39:59,931 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:59,931 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b543b351fe95:0, corePoolSize=2, maxPoolSize=2 2024-12-06T05:39:59,931 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:39:59,933 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=17, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure b543b351fe95,44975,1733463597882, splitWal=true, meta=false 2024-12-06T05:39:59,937 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733463629937 2024-12-06T05:39:59,937 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-06T05:39:59,938 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-06T05:39:59,938 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-06T05:39:59,938 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-06T05:39:59,938 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-06T05:39:59,938 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-06T05:39:59,938 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:59,939 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-06T05:39:59,939 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-06T05:39:59,939 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-06T05:39:59,940 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-06T05:39:59,940 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-06T05:39:59,942 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.large.0-1733463599940,5,FailOnTimeoutGroup] 2024-12-06T05:39:59,942 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.small.0-1733463599942,5,FailOnTimeoutGroup] 2024-12-06T05:39:59,942 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:59,942 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-06T05:39:59,942 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:59,942 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:59,943 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733463599943, completionTime=-1 2024-12-06T05:39:59,943 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-06T05:39:59,943 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-06T05:39:59,943 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T05:39:59,943 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b543b351fe95,42121,1733463596109, seqNum=-1] 2024-12-06T05:39:59,943 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T05:39:59,945 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60091, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T05:39:59,949 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e5ef5686145b3308e84acf2485afdef, regionState=OPEN, lastHost=b543b351fe95,44975,1733463597882, regionLocation=b543b351fe95,44975,1733463597882, openSeqNum=2 2024-12-06T05:39:59,949 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59e2fa4009ebd132f92d2a7d7e292ba0, regionState=OPEN, lastHost=b543b351fe95,42121,1733463596109, regionLocation=b543b351fe95,42121,1733463596109, openSeqNum=2 2024-12-06T05:39:59,950 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad0165368b2f0ee58b25d3d51e5286d5, regionState=OPEN, lastHost=b543b351fe95,43157,1733463596184, regionLocation=b543b351fe95,43157,1733463596184, openSeqNum=2 2024-12-06T05:39:59,950 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71150a984937a6ec0a3a8eb3ffafae8c, regionState=OPEN, lastHost=b543b351fe95,42121,1733463596109, regionLocation=b543b351fe95,42121,1733463596109, openSeqNum=2 2024-12-06T05:39:59,950 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] assignment.RegionStateStore(171): Load hbase:meta entry region=91b02cf9f147172bb99e96b9a5ebc6c0, regionState=OPEN, lastHost=b543b351fe95,44975,1733463597882, regionLocation=b543b351fe95,44975,1733463597882, openSeqNum=2 2024-12-06T05:39:59,951 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f711afceea9ea8582107efd7e8aa74eb, regionState=OPEN, lastHost=b543b351fe95,43157,1733463596184, regionLocation=b543b351fe95,43157,1733463596184, openSeqNum=2 2024-12-06T05:39:59,951 INFO [master/b543b351fe95:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-06T05:39:59,951 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733463659951 2024-12-06T05:39:59,951 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733463719951 2024-12-06T05:39:59,951 INFO [master/b543b351fe95:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-06T05:39:59,951 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): b543b351fe95,44975,1733463597882 had 2 regions 2024-12-06T05:39:59,951 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,42551,1733463595943-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:59,951 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,42551,1733463595943-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:59,951 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,42551,1733463595943-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:59,952 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b543b351fe95:42551, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:59,952 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:59,952 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-06T05:39:59,952 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=17, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure b543b351fe95,44975,1733463597882, splitWal=true, meta=false, isMeta: false 2024-12-06T05:39:59,957 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,44975,1733463597882-splitting 2024-12-06T05:39:59,959 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.776sec 2024-12-06T05:39:59,959 INFO [master/b543b351fe95:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-06T05:39:59,959 INFO [master/b543b351fe95:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-06T05:39:59,959 INFO [master/b543b351fe95:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-06T05:39:59,959 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-06T05:39:59,959 INFO [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-06T05:39:59,960 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,42551,1733463595943-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T05:39:59,960 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,42551,1733463595943-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-06T05:39:59,964 INFO [PEWorker-2 {}] master.SplitWALManager(105): b543b351fe95,44975,1733463597882 WAL count=1, meta=false 2024-12-06T05:39:59,964 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure b543b351fe95%2C44975%2C1733463597882.1733463598114}] 2024-12-06T05:39:59,965 WARN [master/b543b351fe95:0.Chore.1 {}] janitor.CatalogJanitor(177): unknown_server=b543b351fe95,44975,1733463597882/testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., unknown_server=b543b351fe95,44975,1733463597882/testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1733463598004.91b02cf9f147172bb99e96b9a5ebc6c0. 2024-12-06T05:39:59,967 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=b543b351fe95,42121,1733463596109 2024-12-06T05:39:59,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure b543b351fe95%2C44975%2C1733463597882.1733463598114, worker=b543b351fe95,42121,1733463596109}] 2024-12-06T05:39:59,977 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-06T05:39:59,977 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-06T05:39:59,977 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,42551,1733463595943-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:00,049 INFO [regionserver/b543b351fe95:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-06T05:40:00,049 INFO [regionserver/b543b351fe95:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-06T05:40:00,122 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T05:40:00,124 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36995, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T05:40:00,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-12-06T05:40:00,140 INFO [RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,44975,1733463597882-splitting/b543b351fe95%2C44975%2C1733463597882.1733463598114, size=1.6 K (1682bytes) 2024-12-06T05:40:00,140 INFO [RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,44975,1733463597882-splitting/b543b351fe95%2C44975%2C1733463597882.1733463598114 2024-12-06T05:40:00,141 INFO [RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,44975,1733463597882-splitting/b543b351fe95%2C44975%2C1733463597882.1733463598114 after 0ms 2024-12-06T05:40:00,143 INFO [RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,44975,1733463597882-splitting/b543b351fe95%2C44975%2C1733463597882.1733463598114 took 3ms 2024-12-06T05:40:00,150 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for 91b02cf9f147172bb99e96b9a5ebc6c0: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "col" sequence_id: 18446744073709551615 } 2024-12-06T05:40:00,152 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for 4e5ef5686145b3308e84acf2485afdef: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "col" sequence_id: 3 } 2024-12-06T05:40:00,152 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,44975,1733463597882-splitting/b543b351fe95%2C44975%2C1733463597882.1733463598114 so closing down 2024-12-06T05:40:00,152 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-06T05:40:00,156 INFO [RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-b543b351fe95%2C44975%2C1733463597882.1733463598114.temp 2024-12-06T05:40:00,158 INFO [RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/4e5ef5686145b3308e84acf2485afdef/recovered.edits/0000000000000000004-b543b351fe95%2C44975%2C1733463597882.1733463598114.temp 2024-12-06T05:40:00,159 INFO [RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-12-06T05:40:00,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741904_1081 (size=903) 2024-12-06T05:40:00,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741904_1081 (size=903) 2024-12-06T05:40:00,167 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/4e5ef5686145b3308e84acf2485afdef/recovered.edits/0000000000000000004-b543b351fe95%2C44975%2C1733463597882.1733463598114.temp (wrote 6 edits, skipped 0 edits in 0 ms) 2024-12-06T05:40:00,168 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/4e5ef5686145b3308e84acf2485afdef/recovered.edits/0000000000000000004-b543b351fe95%2C44975%2C1733463597882.1733463598114.temp to hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/4e5ef5686145b3308e84acf2485afdef/recovered.edits/0000000000000000009 2024-12-06T05:40:00,168 INFO [RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 8 edits across 1 Regions in 24 ms; skipped=2; WAL=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,44975,1733463597882-splitting/b543b351fe95%2C44975%2C1733463597882.1733463598114, size=1.6 K, length=1682, corrupted=false, cancelled=false 2024-12-06T05:40:00,168 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,44975,1733463597882-splitting/b543b351fe95%2C44975%2C1733463597882.1733463598114, journal: Splitting hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,44975,1733463597882-splitting/b543b351fe95%2C44975%2C1733463597882.1733463598114, size=1.6 K (1682bytes) at 1733463600140Finishing writing output for hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,44975,1733463597882-splitting/b543b351fe95%2C44975%2C1733463597882.1733463598114 so closing down at 1733463600152 (+12 ms)Creating recovered edits writer path=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/4e5ef5686145b3308e84acf2485afdef/recovered.edits/0000000000000000004-b543b351fe95%2C44975%2C1733463597882.1733463598114.temp at 1733463600158 (+6 ms)3 split writer threads finished at 1733463600159 (+1 ms)Closed recovered edits writer path=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/4e5ef5686145b3308e84acf2485afdef/recovered.edits/0000000000000000004-b543b351fe95%2C44975%2C1733463597882.1733463598114.temp (wrote 6 edits, skipped 0 edits in 0 ms) at 1733463600167 (+8 ms)Rename recovered edits hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/4e5ef5686145b3308e84acf2485afdef/recovered.edits/0000000000000000004-b543b351fe95%2C44975%2C1733463597882.1733463598114.temp to hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/4e5ef5686145b3308e84acf2485afdef/recovered.edits/0000000000000000009 at 1733463600168 (+1 ms)Processed 8 edits across 1 Regions in 24 ms; skipped=2; WAL=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,44975,1733463597882-splitting/b543b351fe95%2C44975%2C1733463597882.1733463598114, size=1.6 K, length=1682, corrupted=false, cancelled=false at 1733463600168 2024-12-06T05:40:00,168 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,44975,1733463597882-splitting/b543b351fe95%2C44975%2C1733463597882.1733463598114 2024-12-06T05:40:00,168 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-12-06T05:40:00,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42551 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-12-06T05:40:00,173 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,44975,1733463597882-splitting/b543b351fe95%2C44975%2C1733463597882.1733463598114 to hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/oldWALs 2024-12-06T05:40:00,176 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-12-06T05:40:00,176 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b543b351fe95%2C44975%2C1733463597882.1733463598114, worker=b543b351fe95,42121,1733463596109 in 204 msec 2024-12-06T05:40:00,177 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=b543b351fe95,42121,1733463596109 2024-12-06T05:40:00,180 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-06T05:40:00,180 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; SplitWALProcedure b543b351fe95%2C44975%2C1733463597882.1733463598114, worker=b543b351fe95,42121,1733463596109 in 213 msec 2024-12-06T05:40:00,182 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,44975,1733463597882-splitting dir is empty, no logs to split. 2024-12-06T05:40:00,182 INFO [PEWorker-2 {}] master.SplitWALManager(105): b543b351fe95,44975,1733463597882 WAL count=0, meta=false 2024-12-06T05:40:00,182 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if b543b351fe95,44975,1733463597882 WAL splitting is done? wals=0, meta=false 2024-12-06T05:40:00,184 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for b543b351fe95,44975,1733463597882 failed, ignore...File hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,44975,1733463597882-splitting does not exist. 2024-12-06T05:40:00,186 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=4e5ef5686145b3308e84acf2485afdef, ASSIGN}, {pid=21, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=91b02cf9f147172bb99e96b9a5ebc6c0, ASSIGN}] 2024-12-06T05:40:00,188 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=4e5ef5686145b3308e84acf2485afdef, ASSIGN 2024-12-06T05:40:00,188 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=91b02cf9f147172bb99e96b9a5ebc6c0, ASSIGN 2024-12-06T05:40:00,189 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=4e5ef5686145b3308e84acf2485afdef, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-06T05:40:00,189 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=91b02cf9f147172bb99e96b9a5ebc6c0, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-06T05:40:00,340 DEBUG [b543b351fe95:42551 {}] balancer.BalancerClusterState(204): Hosts are {b543b351fe95=0} racks are {/default-rack=0} 2024-12-06T05:40:00,340 DEBUG [b543b351fe95:42551 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-06T05:40:00,340 DEBUG [b543b351fe95:42551 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-06T05:40:00,340 DEBUG [b543b351fe95:42551 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-06T05:40:00,340 DEBUG [b543b351fe95:42551 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-06T05:40:00,340 INFO [b543b351fe95:42551 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-06T05:40:00,340 INFO [b543b351fe95:42551 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-06T05:40:00,340 DEBUG [b543b351fe95:42551 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-06T05:40:00,340 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=91b02cf9f147172bb99e96b9a5ebc6c0, regionState=OPENING, regionLocation=b543b351fe95,43157,1733463596184 2024-12-06T05:40:00,340 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=4e5ef5686145b3308e84acf2485afdef, regionState=OPENING, regionLocation=b543b351fe95,42121,1733463596109 2024-12-06T05:40:00,343 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-19-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=91b02cf9f147172bb99e96b9a5ebc6c0, ASSIGN because future has completed 2024-12-06T05:40:00,344 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 91b02cf9f147172bb99e96b9a5ebc6c0, server=b543b351fe95,43157,1733463596184}] 2024-12-06T05:40:00,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-19-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=4e5ef5686145b3308e84acf2485afdef, ASSIGN because future has completed 2024-12-06T05:40:00,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e5ef5686145b3308e84acf2485afdef, server=b543b351fe95,42121,1733463596109}] 2024-12-06T05:40:00,352 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testLogSplittingAfterMasterRecoveryDueToZKExpiry', row='', locateType=CURRENT is [region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., hostname=b543b351fe95,44975,1733463597882, seqNum=2] 2024-12-06T05:40:00,353 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] ipc.AbstractRpcClient(357): Not trying to connect to b543b351fe95:44975 this server is in the failed servers list 2024-12-06T05:40:00,353 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., hostname=b543b351fe95,44975,1733463597882, seqNum=2 , the old value is region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., hostname=b543b351fe95,44975,1733463597882, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=b543b351fe95:44975 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: b543b351fe95:44975 2024-12-06T05:40:00,353 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., hostname=b543b351fe95,44975,1733463597882, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: b543b351fe95:44975 2024-12-06T05:40:00,353 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., hostname=b543b351fe95,44975,1733463597882, seqNum=2 from cache 2024-12-06T05:40:00,499 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T05:40:00,501 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-18-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41417, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T05:40:00,506 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(132): Open testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef. 2024-12-06T05:40:00,506 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7752): Opening region: {ENCODED => 4e5ef5686145b3308e84acf2485afdef, NAME => 'testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef.', STARTKEY => '', ENDKEY => '1'} 2024-12-06T05:40:00,507 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testLogSplittingAfterMasterRecoveryDueToZKExpiry 4e5ef5686145b3308e84acf2485afdef 2024-12-06T05:40:00,507 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(898): Instantiated testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:40:00,508 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7794): checking encryption for 4e5ef5686145b3308e84acf2485afdef 2024-12-06T05:40:00,508 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7797): checking classloading for 4e5ef5686145b3308e84acf2485afdef 2024-12-06T05:40:00,508 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] handler.AssignRegionHandler(132): Open testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1733463598004.91b02cf9f147172bb99e96b9a5ebc6c0. 2024-12-06T05:40:00,509 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(7752): Opening region: {ENCODED => 91b02cf9f147172bb99e96b9a5ebc6c0, NAME => 'testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1733463598004.91b02cf9f147172bb99e96b9a5ebc6c0.', STARTKEY => '4', ENDKEY => '5'} 2024-12-06T05:40:00,509 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testLogSplittingAfterMasterRecoveryDueToZKExpiry 91b02cf9f147172bb99e96b9a5ebc6c0 2024-12-06T05:40:00,509 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(898): Instantiated testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1733463598004.91b02cf9f147172bb99e96b9a5ebc6c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:40:00,509 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(7794): checking encryption for 91b02cf9f147172bb99e96b9a5ebc6c0 2024-12-06T05:40:00,509 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(7797): checking classloading for 91b02cf9f147172bb99e96b9a5ebc6c0 2024-12-06T05:40:00,510 INFO [StoreOpener-4e5ef5686145b3308e84acf2485afdef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family col of region 4e5ef5686145b3308e84acf2485afdef 2024-12-06T05:40:00,511 INFO [StoreOpener-4e5ef5686145b3308e84acf2485afdef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e5ef5686145b3308e84acf2485afdef columnFamilyName col 2024-12-06T05:40:00,511 INFO [StoreOpener-91b02cf9f147172bb99e96b9a5ebc6c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family col of region 91b02cf9f147172bb99e96b9a5ebc6c0 2024-12-06T05:40:00,511 DEBUG [StoreOpener-4e5ef5686145b3308e84acf2485afdef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:40:00,512 INFO [StoreOpener-4e5ef5686145b3308e84acf2485afdef-1 {}] regionserver.HStore(327): Store=4e5ef5686145b3308e84acf2485afdef/col, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:40:00,512 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1038): replaying wal for 4e5ef5686145b3308e84acf2485afdef 2024-12-06T05:40:00,512 INFO [StoreOpener-91b02cf9f147172bb99e96b9a5ebc6c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 91b02cf9f147172bb99e96b9a5ebc6c0 columnFamilyName col 2024-12-06T05:40:00,512 DEBUG [StoreOpener-91b02cf9f147172bb99e96b9a5ebc6c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:40:00,513 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/4e5ef5686145b3308e84acf2485afdef 2024-12-06T05:40:00,513 INFO [StoreOpener-91b02cf9f147172bb99e96b9a5ebc6c0-1 {}] regionserver.HStore(327): Store=91b02cf9f147172bb99e96b9a5ebc6c0/col, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:40:00,513 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1038): replaying wal for 91b02cf9f147172bb99e96b9a5ebc6c0 2024-12-06T05:40:00,514 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/91b02cf9f147172bb99e96b9a5ebc6c0 2024-12-06T05:40:00,514 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/4e5ef5686145b3308e84acf2485afdef 2024-12-06T05:40:00,515 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/91b02cf9f147172bb99e96b9a5ebc6c0 2024-12-06T05:40:00,515 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1048): stopping wal replay for 91b02cf9f147172bb99e96b9a5ebc6c0 2024-12-06T05:40:00,515 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1060): Cleaning up temporary data for 91b02cf9f147172bb99e96b9a5ebc6c0 2024-12-06T05:40:00,515 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/4e5ef5686145b3308e84acf2485afdef/recovered.edits/0000000000000000009 2024-12-06T05:40:00,517 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1093): writing seq id for 91b02cf9f147172bb99e96b9a5ebc6c0 2024-12-06T05:40:00,517 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1114): Opened 91b02cf9f147172bb99e96b9a5ebc6c0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67213355, jitterRate=0.0015570372343063354}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:40:00,518 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 91b02cf9f147172bb99e96b9a5ebc6c0 2024-12-06T05:40:00,518 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1006): Region open journal for 91b02cf9f147172bb99e96b9a5ebc6c0: Running coprocessor pre-open hook at 1733463600510Writing region info on filesystem at 1733463600510Initializing all the Stores at 1733463600511 (+1 ms)Instantiating store for column family {NAME => 'col', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463600511Cleaning up temporary data from old regions at 1733463600515 (+4 ms)Running coprocessor post-open hooks at 1733463600518 (+3 ms)Region opened successfully at 1733463600518 2024-12-06T05:40:00,519 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegionServer(2236): Post open deploy tasks for testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1733463598004.91b02cf9f147172bb99e96b9a5ebc6c0., pid=22, masterSystemTime=1733463600498 2024-12-06T05:40:00,519 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5793): Applied 6, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=9, path=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/4e5ef5686145b3308e84acf2485afdef/recovered.edits/0000000000000000009 2024-12-06T05:40:00,519 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(2902): Flushing 4e5ef5686145b3308e84acf2485afdef 1/1 column families, dataSize=210 B heapSize=928 B 2024-12-06T05:40:00,521 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegionServer(2266): Finished post open deploy task for testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1733463598004.91b02cf9f147172bb99e96b9a5ebc6c0. 2024-12-06T05:40:00,521 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] handler.AssignRegionHandler(153): Opened testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1733463598004.91b02cf9f147172bb99e96b9a5ebc6c0. 2024-12-06T05:40:00,522 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=91b02cf9f147172bb99e96b9a5ebc6c0, regionState=OPEN, openSeqNum=2, regionLocation=b543b351fe95,43157,1733463596184 2024-12-06T05:40:00,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-19-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 91b02cf9f147172bb99e96b9a5ebc6c0, server=b543b351fe95,43157,1733463596184 because future has completed 2024-12-06T05:40:00,527 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-06T05:40:00,527 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 91b02cf9f147172bb99e96b9a5ebc6c0, server=b543b351fe95,43157,1733463596184 in 181 msec 2024-12-06T05:40:00,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=17, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=91b02cf9f147172bb99e96b9a5ebc6c0, ASSIGN in 341 msec 2024-12-06T05:40:00,534 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/4e5ef5686145b3308e84acf2485afdef/.tmp/col/f73e87f333fc484c843212df8b21aec1 is 39, key is \x00\x00\x00\x00/col:ql/1733463599159/Put/seqid=0 2024-12-06T05:40:00,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741905_1082 (size=5166) 2024-12-06T05:40:00,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741905_1082 (size=5166) 2024-12-06T05:40:00,539 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=210 B at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/4e5ef5686145b3308e84acf2485afdef/.tmp/col/f73e87f333fc484c843212df8b21aec1 2024-12-06T05:40:00,546 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/4e5ef5686145b3308e84acf2485afdef/.tmp/col/f73e87f333fc484c843212df8b21aec1 as hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/4e5ef5686145b3308e84acf2485afdef/col/f73e87f333fc484c843212df8b21aec1 2024-12-06T05:40:00,552 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/4e5ef5686145b3308e84acf2485afdef/col/f73e87f333fc484c843212df8b21aec1, entries=6, sequenceid=9, filesize=5.0 K 2024-12-06T05:40:00,553 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(3140): Finished flush of dataSize ~210 B/210, heapSize ~912 B/912, currentSize=0 B/0 for 4e5ef5686145b3308e84acf2485afdef in 34ms, sequenceid=9, compaction requested=false; wal=null 2024-12-06T05:40:00,553 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testLogSplittingAfterMasterRecoveryDueToZKExpiry' 2024-12-06T05:40:00,554 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/4e5ef5686145b3308e84acf2485afdef/recovered.edits/0000000000000000009 2024-12-06T05:40:00,555 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1048): stopping wal replay for 4e5ef5686145b3308e84acf2485afdef 2024-12-06T05:40:00,555 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1060): Cleaning up temporary data for 4e5ef5686145b3308e84acf2485afdef 2024-12-06T05:40:00,557 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1093): writing seq id for 4e5ef5686145b3308e84acf2485afdef 2024-12-06T05:40:00,559 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testLogSplittingAfterMasterRecoveryDueToZKExpiry/4e5ef5686145b3308e84acf2485afdef/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-06T05:40:00,560 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1114): Opened 4e5ef5686145b3308e84acf2485afdef; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72253335, jitterRate=0.07665859162807465}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:40:00,560 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4e5ef5686145b3308e84acf2485afdef 2024-12-06T05:40:00,560 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1006): Region open journal for 4e5ef5686145b3308e84acf2485afdef: Running coprocessor pre-open hook at 1733463600508Writing region info on filesystem at 1733463600508Initializing all the Stores at 1733463600509 (+1 ms)Instantiating store for column family {NAME => 'col', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463600509Obtaining lock to block concurrent updates at 1733463600519 (+10 ms)Preparing flush snapshotting stores in 4e5ef5686145b3308e84acf2485afdef at 1733463600519Finished memstore snapshotting testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., syncing WAL and waiting on mvcc, flushsize=dataSize=210, getHeapSize=912, getOffHeapSize=0, getCellsCount=6 at 1733463600519Flushing stores of testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef. at 1733463600519Flushing 4e5ef5686145b3308e84acf2485afdef/col: creating writer at 1733463600519Flushing 4e5ef5686145b3308e84acf2485afdef/col: appending metadata at 1733463600534 (+15 ms)Flushing 4e5ef5686145b3308e84acf2485afdef/col: closing flushed file at 1733463600534Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6822a85f: reopening flushed file at 1733463600545 (+11 ms)Finished flush of dataSize ~210 B/210, heapSize ~912 B/912, currentSize=0 B/0 for 4e5ef5686145b3308e84acf2485afdef in 34ms, sequenceid=9, compaction requested=false; wal=null at 1733463600553 (+8 ms)Cleaning up temporary data from old regions at 1733463600555 (+2 ms)Running coprocessor post-open hooks at 1733463600560 (+5 ms)Region opened successfully at 1733463600560 2024-12-06T05:40:00,561 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2236): Post open deploy tasks for testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., pid=23, masterSystemTime=1733463600499 2024-12-06T05:40:00,563 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2266): Finished post open deploy task for testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef. 2024-12-06T05:40:00,563 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(153): Opened testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef. 2024-12-06T05:40:00,564 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=4e5ef5686145b3308e84acf2485afdef, regionState=OPEN, openSeqNum=10, regionLocation=b543b351fe95,42121,1733463596109 2024-12-06T05:40:00,566 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-19-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e5ef5686145b3308e84acf2485afdef, server=b543b351fe95,42121,1733463596109 because future has completed 2024-12-06T05:40:00,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=20 2024-12-06T05:40:00,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 4e5ef5686145b3308e84acf2485afdef, server=b543b351fe95,42121,1733463596109 in 223 msec 2024-12-06T05:40:00,573 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=17 2024-12-06T05:40:00,573 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=17, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testLogSplittingAfterMasterRecoveryDueToZKExpiry, region=4e5ef5686145b3308e84acf2485afdef, ASSIGN in 385 msec 2024-12-06T05:40:00,573 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(291): removed crashed server b543b351fe95,44975,1733463597882 after splitting done 2024-12-06T05:40:00,575 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; ServerCrashProcedure b543b351fe95,44975,1733463597882, splitWal=true, meta=false in 671 msec 2024-12-06T05:40:01,374 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testLogSplittingAfterMasterRecoveryDueToZKExpiry', row='', locateType=CURRENT is [region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., hostname=b543b351fe95,42121,1733463596109, seqNum=10] 2024-12-06T05:40:01,382 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testLogSplittingAfterMasterRecoveryDueToZKExpiry', row='1', locateType=CURRENT is [region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1733463598004.59e2fa4009ebd132f92d2a7d7e292ba0., hostname=b543b351fe95,42121,1733463596109, seqNum=2] 2024-12-06T05:40:01,387 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testLogSplittingAfterMasterRecoveryDueToZKExpiry', row='2', locateType=CURRENT is [region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1733463598004.ad0165368b2f0ee58b25d3d51e5286d5., hostname=b543b351fe95,43157,1733463596184, seqNum=2] 2024-12-06T05:40:01,387 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T05:40:01,389 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-18-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T05:40:01,393 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testLogSplittingAfterMasterRecoveryDueToZKExpiry', row='3', locateType=CURRENT is [region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1733463598004.71150a984937a6ec0a3a8eb3ffafae8c., hostname=b543b351fe95,42121,1733463596109, seqNum=2] 2024-12-06T05:40:01,398 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testLogSplittingAfterMasterRecoveryDueToZKExpiry', row='4', locateType=CURRENT is [region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1733463598004.91b02cf9f147172bb99e96b9a5ebc6c0., hostname=b543b351fe95,43157,1733463596184, seqNum=2] 2024-12-06T05:40:01,403 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testLogSplittingAfterMasterRecoveryDueToZKExpiry', row='5', locateType=CURRENT is [region=testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1733463598004.f711afceea9ea8582107efd7e8aa74eb., hostname=b543b351fe95,43157,1733463596184, seqNum=2] 2024-12-06T05:40:01,405 DEBUG [Time-limited test {}] master.HMaster(3213): Abort called but aborted=true, stopped=true 2024-12-06T05:40:01,405 ERROR [Time-limited test {}] master.HMaster(3218): Master server abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-06T05:40:01,405 ERROR [Time-limited test {}] master.HMaster(3225): ***** ABORTING master b543b351fe95,42551,1733463595943: killAll ***** 2024-12-06T05:40:01,405 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master 'b543b351fe95,42551,1733463595943' ***** 2024-12-06T05:40:01,405 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: Stopped by Time-limited test 2024-12-06T05:40:01,405 INFO [M:1;b543b351fe95:42551 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T05:40:01,405 INFO [M:1;b543b351fe95:42551 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:1;b543b351fe95:42551. 2024-12-06T05:40:01,405 DEBUG [M:1;b543b351fe95:42551 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T05:40:01,405 DEBUG [M:1;b543b351fe95:42551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:40:01,406 INFO [M:1;b543b351fe95:42551 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T05:40:01,406 DEBUG [M:1;b543b351fe95:42551 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-06T05:40:01,406 DEBUG [M:1;b543b351fe95:42551 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-06T05:40:01,406 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-06T05:40:01,406 DEBUG [master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.large.0-1733463599940 {}] cleaner.HFileCleaner(306): Exit Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.large.0-1733463599940,5,FailOnTimeoutGroup] 2024-12-06T05:40:01,406 DEBUG [master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.small.0-1733463599942 {}] cleaner.HFileCleaner(306): Exit Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.small.0-1733463599942,5,FailOnTimeoutGroup] 2024-12-06T05:40:01,406 INFO [M:1;b543b351fe95:42551 {}] hbase.ChoreService(370): Chore service for: master/b543b351fe95:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-06T05:40:01,406 INFO [M:1;b543b351fe95:42551 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T05:40:01,406 DEBUG [M:1;b543b351fe95:42551 {}] master.HMaster(1795): Stopping service threads 2024-12-06T05:40:01,406 INFO [M:1;b543b351fe95:42551 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-06T05:40:01,406 INFO [M:1;b543b351fe95:42551 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-06T05:40:01,407 INFO [M:1;b543b351fe95:42551 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=true 2024-12-06T05:40:01,407 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-06T05:40:01,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42121-0x100ae0a5c730014, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:40:01,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43157-0x100ae0a5c730015, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:40:01,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42121-0x100ae0a5c730014, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:01,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:40:01,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:01,407 DEBUG [M:1;b543b351fe95:42551 {}] zookeeper.ZKUtil(347): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-06T05:40:01,407 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server b543b351fe95,42121,1733463596109: killAll ***** 2024-12-06T05:40:01,407 WARN [M:1;b543b351fe95:42551 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-06T05:40:01,407 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-06T05:40:01,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43157-0x100ae0a5c730015, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:01,408 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43157-0x100ae0a5c730015, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:40:01,408 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-06T05:40:01,408 INFO [M:1;b543b351fe95:42551 {}] master.ServerManager(1134): Rewriting .lastflushedseqids file at: hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/.lastflushedseqids 2024-12-06T05:40:01,408 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42121-0x100ae0a5c730014, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:40:01,409 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-06T05:40:01,409 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-06T05:40:01,410 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-06T05:40:01,411 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 434618784 }, "NonHeapMemoryUsage": { "committed": 167247872, "init": 7667712, "max": -1, "used": 164699664 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 14574, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 57, "ProcessCallTime_min": 0, "ProcessCallTime_max": 7, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 2, "ProcessCallTime_75th_percentile": 3, "ProcessCallTime_90th_percentile": 4, "ProcessCallTime_95th_percentile": 6, "ProcessCallTime_98th_percentile": 7, "ProcessCallTime_99th_percentile": 7, "ProcessCallTime_99.9th_percentile": 7, "ProcessCallTime_TimeRangeCount_0-1": 11, "ProcessCallTime_TimeRangeCount_1-3": 12, "ProcessCallTime_TimeRangeCount_3-10": 5, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 57, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 1, "QueueCallTime_95th_percentile": 1, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 28, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 57, "TotalCallTime_min": 0, "TotalCallTime_max": 7, "TotalCallTime_mean": 2, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 3, "TotalCallTime_90th_percentile": 4, "TotalCallTime_95th_percentile": 6, "TotalCallTime_98th_percentile": 7, "TotalCallTime_99th_percentile": 7, "TotalCallTime_99.9th_percentile": 7, "TotalCallTime_TimeRangeCount_0-1": 10, "TotalCallTime_TimeRangeCount_1-3": 13, "TotalCallTime_TimeRangeCount_3-10": 5, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 57, "ResponseSize_min": 0, "ResponseSize_max": 1179, "ResponseSize_mean": 104, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 23, "ResponseSize_90th_percentile": 175, "ResponseSize_95th_percentile": 532, "ResponseSize_98th_percentile": 920, "ResponseSize_99th_percentile": 1049, "ResponseSize_99.9th_percentile": 1166, "ResponseSize_SizeRangeCount_0-10": 19, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 5, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 57, "RequestSize_min": 89, "RequestSize_max": 402, "RequestSize_mean": 175, "RequestSize_25th_percentile": 110, "RequestSize_median": 125, "RequestSize_75th_percentile": 225, "RequestSize_90th_percentile": 342, "RequestSize_95th_percentile": 360, "RequestSize_98th_percentile": 385, "RequestSize_99th_percentile": 393, "RequestSize_99.9th_percentile": 401, "RequestSize_SizeRangeCount_10-100": 5, "RequestSize_SizeRangeCount_100-1000": 23, "sentBytes": 4543 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:58086", "tag.serverName": "b543b351fe95,37775,1733463581162", "tag.clusterId": "f24feb01-2567-4d5e-a4c6-86f800bb290a", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95", "regionCount": 0, "storeCount": 0, "hlogFileCount": 4, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733463581162, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 77674, "localBytesRead": 77674, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 30370, "ByteBuffAllocatorPoolAllocationBytes": 399360, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 18, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 8, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 8, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 8, "ScanTime_min": 1, "ScanTime_max": 1, "ScanTime_mean": 1, "ScanTime_25th_percentile": 1, "ScanTime_median": 1, "ScanTime_75th_percentile": 1, "ScanTime_90th_percentile": 1, "ScanTime_95th_percentile": 1, "ScanTime_98th_percentile": 1, "ScanTime_99th_percentile": 1, "ScanTime_99.9th_percentile": 1, "ScanTime_TimeRangeCount_0-1": 1, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 8, "Put_min": 2, "Put_max": 3, "Put_mean": 2, "Put_25th_percentile": 2, "Put_median": 2, "Put_75th_percentile": 3, "Put_90th_percentile": 3, "Put_95th_percentile": 3, "Put_98th_percentile": 3, "Put_99th_percentile": 3, "Put_99.9th_percentile": 3, "Put_TimeRangeCount_1-3": 6, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 8, "ScanSize_min": 1072, "ScanSize_max": 1072, "ScanSize_mean": 1072, "ScanSize_25th_percentile": 1072, "ScanSize_median": 1072, "ScanSize_75th_percentile": 1072, "ScanSize_90th_percentile": 1072, "ScanSize_95th_percentile": 1072, "ScanSize_98th_percentile": 1072, "ScanSize_99th_percentile": 1072, "ScanSize_99.9th_percentile": 1072, "ScanSize_SizeRangeCount_100-1000": 1, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-06T05:40:01,412 DEBUG [HMaster-EventLoopGroup-16-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b543b351fe95,42551,1733463595943 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:40:01,412 WARN [Time-limited test {}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.regionserver.RegionServerAbortedException): Server b543b351fe95,42551,1733463595943 aborting at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:?] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:?] at org.apache.hadoop.hbase.Abortable.abort(Abortable.java:45) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster.killAll(SingleProcessHBaseCluster.java:880) ~[test-classes/:?] at org.apache.hadoop.hbase.TestZooKeeper.after(TestZooKeeper.java:107) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: Server b543b351fe95,42551,1733463595943 aborting at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-12-06T05:40:01,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741906_1083 (size=454) 2024-12-06T05:40:01,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741906_1083 (size=454) 2024-12-06T05:40:01,415 INFO [M:1;b543b351fe95:42551 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-06T05:40:01,415 INFO [M:1;b543b351fe95:42551 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=true 2024-12-06T05:40:01,417 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b543b351fe95,42121,1733463596109' ***** 2024-12-06T05:40:01,417 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: killAll 2024-12-06T05:40:01,417 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server b543b351fe95,43157,1733463596184: killAll ***** 2024-12-06T05:40:01,417 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-06T05:40:01,417 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-06T05:40:01,417 INFO [RS:0;b543b351fe95:42121 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T05:40:01,417 INFO [RS:0;b543b351fe95:42121 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-06T05:40:01,417 INFO [RS:0;b543b351fe95:42121 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-06T05:40:01,417 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T05:40:01,417 INFO [RS:0;b543b351fe95:42121 {}] regionserver.HRegionServer(3091): Received CLOSE for 71150a984937a6ec0a3a8eb3ffafae8c 2024-12-06T05:40:01,417 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-06T05:40:01,418 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-06T05:40:01,418 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-06T05:40:01,419 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 436715936 }, "NonHeapMemoryUsage": { "committed": 167313408, "init": 7667712, "max": -1, "used": 164761424 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 14574, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 57, "ProcessCallTime_min": 0, "ProcessCallTime_max": 7, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 2, "ProcessCallTime_75th_percentile": 3, "ProcessCallTime_90th_percentile": 4, "ProcessCallTime_95th_percentile": 6, "ProcessCallTime_98th_percentile": 7, "ProcessCallTime_99th_percentile": 7, "ProcessCallTime_99.9th_percentile": 7, "ProcessCallTime_TimeRangeCount_0-1": 11, "ProcessCallTime_TimeRangeCount_1-3": 12, "ProcessCallTime_TimeRangeCount_3-10": 5, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 57, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 1, "QueueCallTime_95th_percentile": 1, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 28, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 57, "TotalCallTime_min": 0, "TotalCallTime_max": 7, "TotalCallTime_mean": 2, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 3, "TotalCallTime_90th_percentile": 4, "TotalCallTime_95th_percentile": 6, "TotalCallTime_98th_percentile": 7, "TotalCallTime_99th_percentile": 7, "TotalCallTime_99.9th_percentile": 7, "TotalCallTime_TimeRangeCount_0-1": 10, "TotalCallTime_TimeRangeCount_1-3": 13, "TotalCallTime_TimeRangeCount_3-10": 5, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 57, "ResponseSize_min": 0, "ResponseSize_max": 1179, "ResponseSize_mean": 104, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 23, "ResponseSize_90th_percentile": 175, "ResponseSize_95th_percentile": 532, "ResponseSize_98th_percentile": 920, "ResponseSize_99th_percentile": 1049, "ResponseSize_99.9th_percentile": 1166, "ResponseSize_SizeRangeCount_0-10": 19, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 5, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 57, "RequestSize_min": 89, "RequestSize_max": 402, "RequestSize_mean": 175, "RequestSize_25th_percentile": 110, "RequestSize_median": 125, "RequestSize_75th_percentile": 225, "RequestSize_90th_percentile": 342, "RequestSize_95th_percentile": 360, "RequestSize_98th_percentile": 385, "RequestSize_99th_percentile": 393, "RequestSize_99.9th_percentile": 401, "RequestSize_SizeRangeCount_10-100": 5, "RequestSize_SizeRangeCount_100-1000": 23, "sentBytes": 4543 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:58086", "tag.serverName": "b543b351fe95,37775,1733463581162", "tag.clusterId": "f24feb01-2567-4d5e-a4c6-86f800bb290a", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95", "regionCount": 0, "storeCount": 0, "hlogFileCount": 4, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733463581162, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 77674, "localBytesRead": 77674, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 30370, "ByteBuffAllocatorPoolAllocationBytes": 399360, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 18, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 8, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 8, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 8, "ScanTime_min": 1, "ScanTime_max": 1, "ScanTime_mean": 1, "ScanTime_25th_percentile": 1, "ScanTime_median": 1, "ScanTime_75th_percentile": 1, "ScanTime_90th_percentile": 1, "ScanTime_95th_percentile": 1, "ScanTime_98th_percentile": 1, "ScanTime_99th_percentile": 1, "ScanTime_99.9th_percentile": 1, "ScanTime_TimeRangeCount_0-1": 1, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 8, "Put_min": 2, "Put_max": 3, "Put_mean": 2, "Put_25th_percentile": 2, "Put_median": 2, "Put_75th_percentile": 3, "Put_90th_percentile": 3, "Put_95th_percentile": 3, "Put_98th_percentile": 3, "Put_99th_percentile": 3, "Put_99.9th_percentile": 3, "Put_TimeRangeCount_1-3": 6, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 8, "ScanSize_min": 1072, "ScanSize_max": 1072, "ScanSize_mean": 1072, "ScanSize_25th_percentile": 1072, "ScanSize_median": 1072, "ScanSize_75th_percentile": 1072, "ScanSize_90th_percentile": 1072, "ScanSize_95th_percentile": 1072, "ScanSize_98th_percentile": 1072, "ScanSize_99th_percentile": 1072, "ScanSize_99.9th_percentile": 1072, "ScanSize_SizeRangeCount_100-1000": 1, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-06T05:40:01,420 INFO [RS:0;b543b351fe95:42121 {}] regionserver.HRegionServer(3091): Received CLOSE for 4e5ef5686145b3308e84acf2485afdef 2024-12-06T05:40:01,420 INFO [RS:0;b543b351fe95:42121 {}] regionserver.HRegionServer(3091): Received CLOSE for 59e2fa4009ebd132f92d2a7d7e292ba0 2024-12-06T05:40:01,420 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 71150a984937a6ec0a3a8eb3ffafae8c, disabling compactions & flushes 2024-12-06T05:40:01,420 INFO [RS:0;b543b351fe95:42121 {}] regionserver.HRegionServer(956): aborting server b543b351fe95,42121,1733463596109 2024-12-06T05:40:01,420 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1733463598004.71150a984937a6ec0a3a8eb3ffafae8c. 2024-12-06T05:40:01,420 INFO [RS:0;b543b351fe95:42121 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T05:40:01,420 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1733463598004.71150a984937a6ec0a3a8eb3ffafae8c. 2024-12-06T05:40:01,420 INFO [RS:0;b543b351fe95:42121 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;b543b351fe95:42121. 2024-12-06T05:40:01,420 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1733463598004.71150a984937a6ec0a3a8eb3ffafae8c. after waiting 0 ms 2024-12-06T05:40:01,420 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1733463598004.71150a984937a6ec0a3a8eb3ffafae8c. 2024-12-06T05:40:01,420 DEBUG [RS:0;b543b351fe95:42121 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T05:40:01,420 DEBUG [RS:0;b543b351fe95:42121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:40:01,420 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b543b351fe95,42551,1733463595943 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:40:01,421 INFO [RS:0;b543b351fe95:42121 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T05:40:01,421 INFO [RS:0;b543b351fe95:42121 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T05:40:01,421 INFO [RS:0;b543b351fe95:42121 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T05:40:01,421 INFO [RS:0;b543b351fe95:42121 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-06T05:40:01,421 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1733463598004.71150a984937a6ec0a3a8eb3ffafae8c. 2024-12-06T05:40:01,421 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 71150a984937a6ec0a3a8eb3ffafae8c: Waiting for close lock at 1733463601420Running coprocessor pre-close hooks at 1733463601420Disabling compacts and flushes for region at 1733463601420Disabling writes for close at 1733463601420Writing region close event to WAL at 1733463601421 (+1 ms)Running coprocessor post-close hooks at 1733463601421Closed at 1733463601421 2024-12-06T05:40:01,421 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,3,1733463598004.71150a984937a6ec0a3a8eb3ffafae8c. 2024-12-06T05:40:01,421 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4e5ef5686145b3308e84acf2485afdef, disabling compactions & flushes 2024-12-06T05:40:01,421 WARN [Time-limited test {}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.regionserver.RegionServerAbortedException): Server b543b351fe95,42551,1733463595943 aborting at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:?] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:?] at org.apache.hadoop.hbase.Abortable.abort(Abortable.java:45) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster.killAll(SingleProcessHBaseCluster.java:880) ~[test-classes/:?] at org.apache.hadoop.hbase.TestZooKeeper.after(TestZooKeeper.java:107) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: Server b543b351fe95,42551,1733463595943 aborting at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-12-06T05:40:01,421 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef. 2024-12-06T05:40:01,421 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef. 2024-12-06T05:40:01,421 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef. after waiting 0 ms 2024-12-06T05:40:01,421 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef. 2024-12-06T05:40:01,422 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b543b351fe95,43157,1733463596184' ***** 2024-12-06T05:40:01,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741900_1077 (size=30007) 2024-12-06T05:40:01,422 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: killAll 2024-12-06T05:40:01,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741900_1077 (size=30007) 2024-12-06T05:40:01,422 DEBUG [Time-limited test {}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: killAll 2024-12-06T05:40:01,422 INFO [RS:1;b543b351fe95:43157 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T05:40:01,422 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T05:40:01,422 INFO [RS:1;b543b351fe95:43157 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-06T05:40:01,422 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T05:40:01,422 INFO [RS:1;b543b351fe95:43157 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-06T05:40:01,422 INFO [RS:1;b543b351fe95:43157 {}] regionserver.HRegionServer(3091): Received CLOSE for 91b02cf9f147172bb99e96b9a5ebc6c0 2024-12-06T05:40:01,423 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.TestZooKeeper.after(TestZooKeeper.java:110) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T05:40:01,423 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:40:01,423 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6af7b1db to 127.0.0.1:58086 2024-12-06T05:40:01,423 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-06T05:40:01,423 DEBUG [M:1;b543b351fe95:42551 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T05:40:01,423 INFO [RS:1;b543b351fe95:43157 {}] regionserver.HRegionServer(3091): Received CLOSE for f711afceea9ea8582107efd7e8aa74eb 2024-12-06T05:40:01,423 INFO [M:1;b543b351fe95:42551 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:40:01,423 INFO [RS:1;b543b351fe95:43157 {}] regionserver.HRegionServer(3091): Received CLOSE for ad0165368b2f0ee58b25d3d51e5286d5 2024-12-06T05:40:01,423 DEBUG [M:1;b543b351fe95:42551 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:40:01,423 DEBUG [M:1;b543b351fe95:42551 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T05:40:01,423 INFO [RS:1;b543b351fe95:43157 {}] regionserver.HRegionServer(956): aborting server b543b351fe95,43157,1733463596184 2024-12-06T05:40:01,423 DEBUG [M:1;b543b351fe95:42551 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:40:01,424 INFO [RS:1;b543b351fe95:43157 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T05:40:01,424 INFO [RS:1;b543b351fe95:43157 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;b543b351fe95:43157. 2024-12-06T05:40:01,424 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 91b02cf9f147172bb99e96b9a5ebc6c0, disabling compactions & flushes 2024-12-06T05:40:01,424 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1733463598004.91b02cf9f147172bb99e96b9a5ebc6c0. 2024-12-06T05:40:01,424 DEBUG [RS:1;b543b351fe95:43157 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T05:40:01,424 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1733463598004.91b02cf9f147172bb99e96b9a5ebc6c0. 2024-12-06T05:40:01,424 DEBUG [RS:1;b543b351fe95:43157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:40:01,424 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1733463598004.91b02cf9f147172bb99e96b9a5ebc6c0. after waiting 0 ms 2024-12-06T05:40:01,424 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1733463598004.91b02cf9f147172bb99e96b9a5ebc6c0. 2024-12-06T05:40:01,424 INFO [RS:1;b543b351fe95:43157 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-12-06T05:40:01,424 DEBUG [RS:1;b543b351fe95:43157 {}] regionserver.HRegionServer(1325): Online Regions={91b02cf9f147172bb99e96b9a5ebc6c0=testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1733463598004.91b02cf9f147172bb99e96b9a5ebc6c0., f711afceea9ea8582107efd7e8aa74eb=testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1733463598004.f711afceea9ea8582107efd7e8aa74eb., ad0165368b2f0ee58b25d3d51e5286d5=testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1733463598004.ad0165368b2f0ee58b25d3d51e5286d5.} 2024-12-06T05:40:01,424 DEBUG [RS:1;b543b351fe95:43157 {}] regionserver.HRegionServer(1351): Waiting on 91b02cf9f147172bb99e96b9a5ebc6c0, ad0165368b2f0ee58b25d3d51e5286d5, f711afceea9ea8582107efd7e8aa74eb 2024-12-06T05:40:01,424 INFO [RS:0;b543b351fe95:42121 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-12-06T05:40:01,424 DEBUG [RS:0;b543b351fe95:42121 {}] regionserver.HRegionServer(1325): Online Regions={4e5ef5686145b3308e84acf2485afdef=testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef., 1588230740=hbase:meta,,1.1588230740, 59e2fa4009ebd132f92d2a7d7e292ba0=testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1733463598004.59e2fa4009ebd132f92d2a7d7e292ba0.} 2024-12-06T05:40:01,424 DEBUG [RS:0;b543b351fe95:42121 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 4e5ef5686145b3308e84acf2485afdef, 59e2fa4009ebd132f92d2a7d7e292ba0 2024-12-06T05:40:01,424 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T05:40:01,424 INFO [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T05:40:01,424 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T05:40:01,424 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T05:40:01,424 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T05:40:01,424 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef. 2024-12-06T05:40:01,424 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4e5ef5686145b3308e84acf2485afdef: Waiting for close lock at 1733463601421Running coprocessor pre-close hooks at 1733463601421Disabling compacts and flushes for region at 1733463601421Disabling writes for close at 1733463601421Writing region close event to WAL at 1733463601424 (+3 ms)Running coprocessor post-close hooks at 1733463601424Closed at 1733463601424 2024-12-06T05:40:01,425 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,,1733463598004.4e5ef5686145b3308e84acf2485afdef. 2024-12-06T05:40:01,425 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1733463598004.91b02cf9f147172bb99e96b9a5ebc6c0. 2024-12-06T05:40:01,425 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 59e2fa4009ebd132f92d2a7d7e292ba0, disabling compactions & flushes 2024-12-06T05:40:01,425 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 91b02cf9f147172bb99e96b9a5ebc6c0: Waiting for close lock at 1733463601424Running coprocessor pre-close hooks at 1733463601424Disabling compacts and flushes for region at 1733463601424Disabling writes for close at 1733463601424Writing region close event to WAL at 1733463601425 (+1 ms)Running coprocessor post-close hooks at 1733463601425Closed at 1733463601425 2024-12-06T05:40:01,425 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1733463598004.59e2fa4009ebd132f92d2a7d7e292ba0. 2024-12-06T05:40:01,425 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1733463598004.59e2fa4009ebd132f92d2a7d7e292ba0. 2024-12-06T05:40:01,425 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1733463598004.59e2fa4009ebd132f92d2a7d7e292ba0. after waiting 0 ms 2024-12-06T05:40:01,425 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1733463598004.59e2fa4009ebd132f92d2a7d7e292ba0. 2024-12-06T05:40:01,425 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,4,1733463598004.91b02cf9f147172bb99e96b9a5ebc6c0. 2024-12-06T05:40:01,425 ERROR [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 12622 in region hbase:meta,,1.1588230740 2024-12-06T05:40:01,425 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f711afceea9ea8582107efd7e8aa74eb, disabling compactions & flushes 2024-12-06T05:40:01,425 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1733463598004.f711afceea9ea8582107efd7e8aa74eb. 2024-12-06T05:40:01,425 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-06T05:40:01,425 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1733463598004.f711afceea9ea8582107efd7e8aa74eb. 2024-12-06T05:40:01,425 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1733463598004.f711afceea9ea8582107efd7e8aa74eb. after waiting 0 ms 2024-12-06T05:40:01,425 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1733463598004.f711afceea9ea8582107efd7e8aa74eb. 2024-12-06T05:40:01,425 INFO [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T05:40:01,425 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733463601424Running coprocessor pre-close hooks at 1733463601424Disabling compacts and flushes for region at 1733463601424Disabling writes for close at 1733463601424Writing region close event to WAL at 1733463601425 (+1 ms)Running coprocessor post-close hooks at 1733463601425Closed at 1733463601425 2024-12-06T05:40:01,425 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-06T05:40:01,425 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1733463598004.f711afceea9ea8582107efd7e8aa74eb. 2024-12-06T05:40:01,425 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f711afceea9ea8582107efd7e8aa74eb: Waiting for close lock at 1733463601425Running coprocessor pre-close hooks at 1733463601425Disabling compacts and flushes for region at 1733463601425Disabling writes for close at 1733463601425Writing region close event to WAL at 1733463601425Running coprocessor post-close hooks at 1733463601425Closed at 1733463601425 2024-12-06T05:40:01,425 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,5,1733463598004.f711afceea9ea8582107efd7e8aa74eb. 2024-12-06T05:40:01,426 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ad0165368b2f0ee58b25d3d51e5286d5, disabling compactions & flushes 2024-12-06T05:40:01,426 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1733463598004.ad0165368b2f0ee58b25d3d51e5286d5. 2024-12-06T05:40:01,426 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1733463598004.ad0165368b2f0ee58b25d3d51e5286d5. 2024-12-06T05:40:01,426 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1733463598004.ad0165368b2f0ee58b25d3d51e5286d5. after waiting 0 ms 2024-12-06T05:40:01,426 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1733463598004.ad0165368b2f0ee58b25d3d51e5286d5. 2024-12-06T05:40:01,426 ERROR [M:1;b543b351fe95:42551 {}] regionserver.HRegion(1960): Memstore data size is 26096 in region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:40:01,426 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1733463598004.59e2fa4009ebd132f92d2a7d7e292ba0. 2024-12-06T05:40:01,426 INFO [M:1;b543b351fe95:42551 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:40:01,426 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 59e2fa4009ebd132f92d2a7d7e292ba0: Waiting for close lock at 1733463601425Running coprocessor pre-close hooks at 1733463601425Disabling compacts and flushes for region at 1733463601425Disabling writes for close at 1733463601425Writing region close event to WAL at 1733463601426 (+1 ms)Running coprocessor post-close hooks at 1733463601426Closed at 1733463601426 2024-12-06T05:40:01,426 DEBUG [M:1;b543b351fe95:42551 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733463601423Disabling compacts and flushes for region at 1733463601423Disabling writes for close at 1733463601423Writing region close event to WAL at 1733463601426 (+3 ms)Closed at 1733463601426 2024-12-06T05:40:01,426 INFO [M:1;b543b351fe95:42551 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-06T05:40:01,426 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,1,1733463598004.59e2fa4009ebd132f92d2a7d7e292ba0. 2024-12-06T05:40:01,426 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T05:40:01,426 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1733463598004.ad0165368b2f0ee58b25d3d51e5286d5. 2024-12-06T05:40:01,426 INFO [M:1;b543b351fe95:42551 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42551 2024-12-06T05:40:01,426 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ad0165368b2f0ee58b25d3d51e5286d5: Waiting for close lock at 1733463601426Running coprocessor pre-close hooks at 1733463601426Disabling compacts and flushes for region at 1733463601426Disabling writes for close at 1733463601426Writing region close event to WAL at 1733463601426Running coprocessor post-close hooks at 1733463601426Closed at 1733463601426 2024-12-06T05:40:01,426 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testLogSplittingAfterMasterRecoveryDueToZKExpiry,2,1733463598004.ad0165368b2f0ee58b25d3d51e5286d5. 2024-12-06T05:40:01,426 INFO [M:1;b543b351fe95:42551 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T05:40:01,464 INFO [regionserver/b543b351fe95:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T05:40:01,497 INFO [regionserver/b543b351fe95:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T05:40:01,528 INFO [M:1;b543b351fe95:42551 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T05:40:01,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:40:01,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42551-0x100ae0a5c730013, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:40:01,624 INFO [RS:1;b543b351fe95:43157 {}] regionserver.HRegionServer(976): stopping server b543b351fe95,43157,1733463596184; all regions closed. 2024-12-06T05:40:01,624 INFO [RS:0;b543b351fe95:42121 {}] regionserver.HRegionServer(976): stopping server b543b351fe95,42121,1733463596109; all regions closed. 2024-12-06T05:40:01,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741888_1065 (size=1263) 2024-12-06T05:40:01,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741888_1065 (size=1263) 2024-12-06T05:40:01,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741889_1066 (size=14032) 2024-12-06T05:40:01,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741889_1066 (size=14032) 2024-12-06T05:40:01,630 DEBUG [RS:1;b543b351fe95:43157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:40:01,630 INFO [RS:1;b543b351fe95:43157 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T05:40:01,631 INFO [RS:1;b543b351fe95:43157 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T05:40:01,631 INFO [RS:1;b543b351fe95:43157 {}] hbase.ChoreService(370): Chore service for: regionserver/b543b351fe95:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-06T05:40:01,631 INFO [RS:1;b543b351fe95:43157 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T05:40:01,631 INFO [RS:1;b543b351fe95:43157 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T05:40:01,631 INFO [RS:1;b543b351fe95:43157 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T05:40:01,631 INFO [regionserver/b543b351fe95:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T05:40:01,631 INFO [RS:1;b543b351fe95:43157 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T05:40:01,632 INFO [RS:1;b543b351fe95:43157 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43157 2024-12-06T05:40:01,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43157-0x100ae0a5c730015, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b543b351fe95,43157,1733463596184 2024-12-06T05:40:01,633 INFO [RS:1;b543b351fe95:43157 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T05:40:01,633 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$360/0x00007fef248de4b8@194a9bb4 rejected from java.util.concurrent.ThreadPoolExecutor@1d750c45[Shutting down, pool size = 1, active threads = 0, queued tasks = 0, completed tasks = 21] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1360) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-12-06T05:40:01,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741887_1064 (size=1296) 2024-12-06T05:40:01,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741887_1064 (size=1296) 2024-12-06T05:40:01,635 DEBUG [RS:0;b543b351fe95:42121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:40:01,635 INFO [RS:0;b543b351fe95:42121 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T05:40:01,635 INFO [RS:0;b543b351fe95:42121 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T05:40:01,635 INFO [RS:0;b543b351fe95:42121 {}] hbase.ChoreService(370): Chore service for: regionserver/b543b351fe95:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-06T05:40:01,635 INFO [RS:0;b543b351fe95:42121 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T05:40:01,635 INFO [regionserver/b543b351fe95:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T05:40:01,635 INFO [RS:0;b543b351fe95:42121 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42121 2024-12-06T05:40:01,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42121-0x100ae0a5c730014, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b543b351fe95,42121,1733463596109 2024-12-06T05:40:01,637 INFO [RS:0;b543b351fe95:42121 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T05:40:01,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43157-0x100ae0a5c730015, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:40:01,735 INFO [RS:1;b543b351fe95:43157 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T05:40:01,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43157-0x100ae0a5c730015, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:40:01,735 INFO [RS:1;b543b351fe95:43157 {}] regionserver.HRegionServer(1031): Exiting; stopping=b543b351fe95,43157,1733463596184; zookeeper connection closed. 2024-12-06T05:40:01,735 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6c9bdcd8 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6c9bdcd8 2024-12-06T05:40:01,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42121-0x100ae0a5c730014, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:40:01,738 INFO [RS:0;b543b351fe95:42121 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T05:40:01,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42121-0x100ae0a5c730014, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:40:01,738 INFO [RS:0;b543b351fe95:42121 {}] regionserver.HRegionServer(1031): Exiting; stopping=b543b351fe95,42121,1733463596109; zookeeper connection closed. 2024-12-06T05:40:01,738 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7ad38fd3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7ad38fd3 2024-12-06T05:40:01,739 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 2 master(s) and 3 regionserver(s) complete 2024-12-06T05:40:01,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730011, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:40:01,842 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:40:01,849 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=testing utility connecting to ZooKeeper ensemble=127.0.0.1:58086 2024-12-06T05:40:01,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility0x0, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:40:01,852 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): testing utility-0x100ae0a5c730018 connected 2024-12-06T05:40:01,853 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-06T05:40:01,854 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/meta-region-server 2024-12-06T05:40:01,855 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/rs 2024-12-06T05:40:01,856 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/splitWAL 2024-12-06T05:40:01,856 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters 2024-12-06T05:40:01,857 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc 2024-12-06T05:40:01,858 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/draining 2024-12-06T05:40:01,859 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/master-maintenance 2024-12-06T05:40:01,860 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot 2024-12-06T05:40:01,861 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/hbaseid 2024-12-06T05:40:01,862 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/table 2024-12-06T05:40:01,867 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc/abort 2024-12-06T05:40:01,868 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc/acquired 2024-12-06T05:40:01,868 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc/reached 2024-12-06T05:40:01,871 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot/abort 2024-12-06T05:40:01,872 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot/acquired 2024-12-06T05:40:01,872 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot/reached 2024-12-06T05:40:01,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/reached 2024-12-06T05:40:01,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/online-snapshot 2024-12-06T05:40:01,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/acquired 2024-12-06T05:40:01,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/abort 2024-12-06T05:40:01,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/reached 2024-12-06T05:40:01,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/flush-table-proc 2024-12-06T05:40:01,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/acquired 2024-12-06T05:40:01,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/abort 2024-12-06T05:40:01,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/table 2024-12-06T05:40:01,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:01,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/hbaseid 2024-12-06T05:40:01,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot 2024-12-06T05:40:01,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master-maintenance 2024-12-06T05:40:01,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/draining 2024-12-06T05:40:01,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc 2024-12-06T05:40:01,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:40:01,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/splitWAL 2024-12-06T05:40:01,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs 2024-12-06T05:40:01,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:40:01,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T05:40:01,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase 2024-12-06T05:40:01,895 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestZooKeeper#testLogSplittingAfterMasterRecoveryDueToZKExpiry Thread=390 (was 345) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-18-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (268780456) connection to localhost/127.0.0.1:34453 from jenkins.hfs.6 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:34453 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-18-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-19-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/b543b351fe95:0.Chore.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-16-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: BootstrapNodeManager java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (268780456) connection to localhost/127.0.0.1:34453 from jenkins.hfs.7 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: region-location-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-19-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_477923861_22 at /127.0.0.1:58202 [Waiting for operation #57] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-15-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-16-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-16-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1876695159_22 at /127.0.0.1:57924 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_477923861_22 at /127.0.0.1:47418 [Waiting for operation #43] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (268780456) connection to localhost/127.0.0.1:34453 from jenkins.hfs.8 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-18-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-19-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/b543b351fe95:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/b543b351fe95:0.logRoller java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:180) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-15-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.8@localhost:34453 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-15-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:34453 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=685 (was 632) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=467 (was 456) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4422 (was 4552) 2024-12-06T05:40:01,909 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestZooKeeper#testRegionAssignmentAfterMasterRecoveryDueToZKExpiry Thread=390, OpenFileDescriptor=685, MaxFileDescriptor=1048576, SystemLoadAverage=467, ProcessCount=11, AvailableMemoryMB=4422 2024-12-06T05:40:01,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741907_1084 (size=7) 2024-12-06T05:40:01,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741907_1084 (size=7) 2024-12-06T05:40:01,917 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 with version=8 2024-12-06T05:40:01,917 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/hbase-staging 2024-12-06T05:40:01,920 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b543b351fe95:0 server-side Connection retries=45 2024-12-06T05:40:01,920 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:40:01,920 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T05:40:01,920 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T05:40:01,920 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:40:01,920 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T05:40:01,920 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-06T05:40:01,920 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T05:40:01,921 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46477 2024-12-06T05:40:01,922 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46477 connecting to ZooKeeper ensemble=127.0.0.1:58086 2024-12-06T05:40:01,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:464770x0, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:40:01,924 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46477-0x100ae0a5c730019 connected 2024-12-06T05:40:01,932 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:40:01,934 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:40:01,936 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T05:40:01,936 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279, hbase.cluster.distributed=false 2024-12-06T05:40:01,938 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T05:40:01,938 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46477 2024-12-06T05:40:01,939 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46477 2024-12-06T05:40:01,941 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46477 2024-12-06T05:40:01,941 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46477 2024-12-06T05:40:01,941 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46477 2024-12-06T05:40:01,942 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b543b351fe95:0 server-side Connection retries=45 2024-12-06T05:40:01,942 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:40:01,942 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T05:40:01,942 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T05:40:01,942 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:40:01,942 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T05:40:01,942 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-06T05:40:01,942 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T05:40:01,943 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32775 2024-12-06T05:40:01,944 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:32775 connecting to ZooKeeper ensemble=127.0.0.1:58086 2024-12-06T05:40:01,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:327750x0, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:40:01,946 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:32775-0x100ae0a5c73001a connected 2024-12-06T05:40:01,948 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:40:01,950 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:40:01,952 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T05:40:01,953 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279, hbase.cluster.distributed=false 2024-12-06T05:40:01,954 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T05:40:01,955 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32775 2024-12-06T05:40:01,955 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32775 2024-12-06T05:40:01,955 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32775 2024-12-06T05:40:01,955 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32775 2024-12-06T05:40:01,956 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32775 2024-12-06T05:40:01,969 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b543b351fe95:0 server-side Connection retries=45 2024-12-06T05:40:01,969 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:40:01,969 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T05:40:01,969 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T05:40:01,969 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:40:01,969 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T05:40:01,970 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T05:40:01,970 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T05:40:01,970 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35247 2024-12-06T05:40:01,971 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35247 connecting to ZooKeeper ensemble=127.0.0.1:58086 2024-12-06T05:40:01,972 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:40:01,973 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:40:01,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:352470x0, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:40:01,977 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35247-0x100ae0a5c73001b connected 2024-12-06T05:40:01,977 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35247-0x100ae0a5c73001b, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T05:40:01,977 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T05:40:01,978 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T05:40:01,978 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35247-0x100ae0a5c73001b, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:40:01,979 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35247-0x100ae0a5c73001b, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T05:40:01,979 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35247 2024-12-06T05:40:01,980 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35247 2024-12-06T05:40:01,980 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35247 2024-12-06T05:40:01,980 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35247 2024-12-06T05:40:01,980 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35247 2024-12-06T05:40:01,993 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b543b351fe95:0 server-side Connection retries=45 2024-12-06T05:40:01,993 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:40:01,993 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T05:40:01,993 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T05:40:01,993 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:40:01,994 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T05:40:01,994 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T05:40:01,994 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T05:40:01,994 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38779 2024-12-06T05:40:01,995 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38779 connecting to ZooKeeper ensemble=127.0.0.1:58086 2024-12-06T05:40:01,995 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:40:01,997 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:40:02,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:387790x0, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:40:02,000 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:387790x0, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T05:40:02,000 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38779-0x100ae0a5c73001c connected 2024-12-06T05:40:02,000 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T05:40:02,001 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T05:40:02,001 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38779-0x100ae0a5c73001c, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:40:02,002 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38779-0x100ae0a5c73001c, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T05:40:02,004 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38779 2024-12-06T05:40:02,004 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38779 2024-12-06T05:40:02,005 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38779 2024-12-06T05:40:02,006 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38779 2024-12-06T05:40:02,008 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38779 2024-12-06T05:40:02,018 DEBUG [M:1;b543b351fe95:32775 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:1;b543b351fe95:32775 2024-12-06T05:40:02,018 DEBUG [M:0;b543b351fe95:46477 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b543b351fe95:46477 2024-12-06T05:40:02,018 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b543b351fe95,32775,1733463601942 2024-12-06T05:40:02,019 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b543b351fe95,46477,1733463601919 2024-12-06T05:40:02,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:40:02,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x100ae0a5c73001b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:40:02,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:40:02,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x100ae0a5c73001c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:40:02,020 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b543b351fe95,32775,1733463601942 2024-12-06T05:40:02,020 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b543b351fe95,46477,1733463601919 2024-12-06T05:40:02,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x100ae0a5c73001c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T05:40:02,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x100ae0a5c73001c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:02,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:02,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x100ae0a5c73001b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T05:40:02,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x100ae0a5c73001b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:02,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:02,022 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T05:40:02,022 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b543b351fe95,32775,1733463601942 from backup master directory 2024-12-06T05:40:02,022 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T05:40:02,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b543b351fe95,32775,1733463601942 2024-12-06T05:40:02,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x100ae0a5c73001c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:40:02,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:40:02,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x100ae0a5c73001b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:40:02,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:40:02,023 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ActiveMasterManager(296): Another master is the active master, b543b351fe95,32775,1733463601942; waiting to become the next active master 2024-12-06T05:40:02,025 WARN [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T05:40:02,025 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b543b351fe95,32775,1733463601942 2024-12-06T05:40:02,030 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/hbase.id] with ID: 976c7c09-c307-4348-ab53-202fa67db969 2024-12-06T05:40:02,030 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/.tmp/hbase.id 2024-12-06T05:40:02,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741908_1085 (size=42) 2024-12-06T05:40:02,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741908_1085 (size=42) 2024-12-06T05:40:02,425 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-06T05:40:02,437 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/.tmp/hbase.id]:[hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/hbase.id] 2024-12-06T05:40:02,451 INFO [master/b543b351fe95:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:40:02,451 INFO [master/b543b351fe95:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-06T05:40:02,452 INFO [master/b543b351fe95:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-12-06T05:40:02,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:02,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x100ae0a5c73001c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:02,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:02,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x100ae0a5c73001b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:02,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741909_1086 (size=196) 2024-12-06T05:40:02,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741909_1086 (size=196) 2024-12-06T05:40:02,460 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T05:40:02,460 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-06T05:40:02,461 INFO [master/b543b351fe95:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:40:02,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741910_1087 (size=1189) 2024-12-06T05:40:02,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741910_1087 (size=1189) 2024-12-06T05:40:02,468 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store 2024-12-06T05:40:02,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741911_1088 (size=34) 2024-12-06T05:40:02,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741911_1088 (size=34) 2024-12-06T05:40:02,475 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:40:02,476 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T05:40:02,476 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:40:02,476 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:40:02,476 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T05:40:02,476 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:40:02,476 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:40:02,476 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733463602476Disabling compacts and flushes for region at 1733463602476Disabling writes for close at 1733463602476Writing region close event to WAL at 1733463602476Closed at 1733463602476 2024-12-06T05:40:02,477 WARN [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/.initializing 2024-12-06T05:40:02,477 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,32775,1733463601942 2024-12-06T05:40:02,478 INFO [master/b543b351fe95:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:40:02,480 INFO [master/b543b351fe95:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b543b351fe95%2C32775%2C1733463601942, suffix=, logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,32775,1733463601942, archiveDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/oldWALs, maxLogs=10 2024-12-06T05:40:02,495 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,32775,1733463601942/b543b351fe95%2C32775%2C1733463601942.1733463602480, exclude list is [], retry=0 2024-12-06T05:40:02,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39561,DS-4485eaa6-367b-4084-97e1-b834fb8d8596,DISK] 2024-12-06T05:40:02,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-80ddb16c-ec66-4e39-a44c-0a80fa143251,DISK] 2024-12-06T05:40:02,499 INFO [master/b543b351fe95:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,32775,1733463601942/b543b351fe95%2C32775%2C1733463601942.1733463602480 2024-12-06T05:40:02,499 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43559:43559),(127.0.0.1/127.0.0.1:44855:44855)] 2024-12-06T05:40:02,499 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:40:02,499 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:40:02,500 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:40:02,500 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:40:02,501 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:40:02,502 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-06T05:40:02,502 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:40:02,503 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:40:02,503 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:40:02,504 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-06T05:40:02,504 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:40:02,504 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:40:02,504 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:40:02,505 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-06T05:40:02,505 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:40:02,505 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:40:02,505 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:40:02,506 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-06T05:40:02,506 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:40:02,507 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:40:02,507 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:40:02,507 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:40:02,507 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:40:02,509 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:40:02,509 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:40:02,509 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T05:40:02,510 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:40:02,512 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:40:02,513 INFO [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66465789, jitterRate=-0.009582564234733582}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T05:40:02,513 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733463602500Initializing all the Stores at 1733463602501 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463602501Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463602501Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463602501Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463602501Cleaning up temporary data from old regions at 1733463602509 (+8 ms)Region opened successfully at 1733463602513 (+4 ms) 2024-12-06T05:40:02,513 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-06T05:40:02,516 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ac85a99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b543b351fe95/172.17.0.2:0 2024-12-06T05:40:02,517 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-06T05:40:02,517 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-06T05:40:02,517 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-06T05:40:02,517 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-06T05:40:02,518 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-06T05:40:02,518 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-06T05:40:02,518 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-06T05:40:02,520 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-06T05:40:02,520 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-06T05:40:02,521 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-06T05:40:02,521 INFO [master/b543b351fe95:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-06T05:40:02,522 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-06T05:40:02,522 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-06T05:40:02,522 INFO [master/b543b351fe95:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-06T05:40:02,523 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-06T05:40:02,524 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-06T05:40:02,524 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-06T05:40:02,525 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-06T05:40:02,526 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-06T05:40:02,527 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-06T05:40:02,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x100ae0a5c73001c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T05:40:02,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T05:40:02,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T05:40:02,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x100ae0a5c73001b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T05:40:02,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:02,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x100ae0a5c73001c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:02,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:02,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x100ae0a5c73001b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:02,528 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=b543b351fe95,32775,1733463601942, sessionid=0x100ae0a5c73001a, setting cluster-up flag (Was=false) 2024-12-06T05:40:02,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x100ae0a5c73001b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:02,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:02,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:02,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x100ae0a5c73001c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:02,533 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-06T05:40:02,534 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b543b351fe95,32775,1733463601942 2024-12-06T05:40:02,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:02,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x100ae0a5c73001b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:02,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:02,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x100ae0a5c73001c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:02,538 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-06T05:40:02,539 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b543b351fe95,32775,1733463601942 2024-12-06T05:40:02,540 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-06T05:40:02,542 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-06T05:40:02,542 INFO [master/b543b351fe95:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-06T05:40:02,542 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b543b351fe95,32775,1733463601942 Number of backup masters: 1 b543b351fe95,46477,1733463601919 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-06T05:40:02,543 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:40:02,543 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:40:02,544 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:40:02,544 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b543b351fe95:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:40:02,544 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b543b351fe95:0, corePoolSize=10, maxPoolSize=10 2024-12-06T05:40:02,544 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:02,544 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b543b351fe95:0, corePoolSize=2, maxPoolSize=2 2024-12-06T05:40:02,544 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:02,546 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T05:40:02,546 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733463632546 2024-12-06T05:40:02,546 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-06T05:40:02,546 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-06T05:40:02,546 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-06T05:40:02,546 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-06T05:40:02,546 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-06T05:40:02,546 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-06T05:40:02,546 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-06T05:40:02,547 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:40:02,547 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-06T05:40:02,548 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:02,548 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-06T05:40:02,549 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-06T05:40:02,549 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-06T05:40:02,550 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-06T05:40:02,551 INFO [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-06T05:40:02,553 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.large.0-1733463602551,5,FailOnTimeoutGroup] 2024-12-06T05:40:02,555 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.small.0-1733463602553,5,FailOnTimeoutGroup] 2024-12-06T05:40:02,555 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:02,555 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-06T05:40:02,555 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:02,555 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:02,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741913_1090 (size=1321) 2024-12-06T05:40:02,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741913_1090 (size=1321) 2024-12-06T05:40:02,560 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-06T05:40:02,560 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:40:02,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741914_1091 (size=32) 2024-12-06T05:40:02,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741914_1091 (size=32) 2024-12-06T05:40:02,571 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:40:02,572 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-06T05:40:02,573 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T05:40:02,573 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:40:02,574 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:40:02,574 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-06T05:40:02,575 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T05:40:02,575 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:40:02,576 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:40:02,576 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-06T05:40:02,577 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T05:40:02,577 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:40:02,577 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:40:02,578 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-06T05:40:02,579 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T05:40:02,579 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:40:02,579 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:40:02,579 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T05:40:02,580 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/hbase/meta/1588230740 2024-12-06T05:40:02,580 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/hbase/meta/1588230740 2024-12-06T05:40:02,581 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T05:40:02,581 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T05:40:02,582 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T05:40:02,583 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T05:40:02,584 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:40:02,585 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75334783, jitterRate=0.12257574498653412}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T05:40:02,585 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733463602571Initializing all the Stores at 1733463602572 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463602572Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463602572Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463602572Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463602572Cleaning up temporary data from old regions at 1733463602581 (+9 ms)Region opened successfully at 1733463602585 (+4 ms) 2024-12-06T05:40:02,585 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T05:40:02,585 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T05:40:02,585 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T05:40:02,585 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T05:40:02,585 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T05:40:02,586 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T05:40:02,586 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733463602585Disabling compacts and flushes for region at 1733463602585Disabling writes for close at 1733463602585Writing region close event to WAL at 1733463602586 (+1 ms)Closed at 1733463602586 2024-12-06T05:40:02,588 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T05:40:02,588 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-06T05:40:02,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-06T05:40:02,589 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T05:40:02,590 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-06T05:40:02,610 INFO [RS:1;b543b351fe95:38779 {}] regionserver.HRegionServer(746): ClusterId : 976c7c09-c307-4348-ab53-202fa67db969 2024-12-06T05:40:02,610 INFO [RS:0;b543b351fe95:35247 {}] regionserver.HRegionServer(746): ClusterId : 976c7c09-c307-4348-ab53-202fa67db969 2024-12-06T05:40:02,610 DEBUG [RS:0;b543b351fe95:35247 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T05:40:02,610 DEBUG [RS:1;b543b351fe95:38779 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T05:40:02,611 DEBUG [RS:0;b543b351fe95:35247 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T05:40:02,611 DEBUG [RS:0;b543b351fe95:35247 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T05:40:02,611 DEBUG [RS:1;b543b351fe95:38779 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T05:40:02,611 DEBUG [RS:1;b543b351fe95:38779 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T05:40:02,613 DEBUG [RS:1;b543b351fe95:38779 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T05:40:02,613 DEBUG [RS:0;b543b351fe95:35247 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T05:40:02,613 DEBUG [RS:0;b543b351fe95:35247 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b44f97a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b543b351fe95/172.17.0.2:0 2024-12-06T05:40:02,613 DEBUG [RS:1;b543b351fe95:38779 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e58f88b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b543b351fe95/172.17.0.2:0 2024-12-06T05:40:02,624 DEBUG [RS:0;b543b351fe95:35247 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b543b351fe95:35247 2024-12-06T05:40:02,624 DEBUG [RS:1;b543b351fe95:38779 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;b543b351fe95:38779 2024-12-06T05:40:02,624 INFO [RS:1;b543b351fe95:38779 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T05:40:02,624 INFO [RS:0;b543b351fe95:35247 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T05:40:02,624 INFO [RS:1;b543b351fe95:38779 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T05:40:02,624 DEBUG [RS:1;b543b351fe95:38779 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T05:40:02,624 INFO [RS:0;b543b351fe95:35247 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T05:40:02,624 DEBUG [RS:0;b543b351fe95:35247 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T05:40:02,625 INFO [RS:1;b543b351fe95:38779 {}] regionserver.HRegionServer(2659): reportForDuty to master=b543b351fe95,32775,1733463601942 with port=38779, startcode=1733463601993 2024-12-06T05:40:02,625 INFO [RS:0;b543b351fe95:35247 {}] regionserver.HRegionServer(2659): reportForDuty to master=b543b351fe95,32775,1733463601942 with port=35247, startcode=1733463601969 2024-12-06T05:40:02,625 DEBUG [RS:1;b543b351fe95:38779 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T05:40:02,625 DEBUG [RS:0;b543b351fe95:35247 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T05:40:02,627 INFO [HMaster-EventLoopGroup-21-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33155, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.9 (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T05:40:02,627 INFO [HMaster-EventLoopGroup-21-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51915, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.10 (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T05:40:02,627 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32775 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b543b351fe95,38779,1733463601993 2024-12-06T05:40:02,628 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32775 {}] master.ServerManager(517): Registering regionserver=b543b351fe95,38779,1733463601993 2024-12-06T05:40:02,629 DEBUG [RS:1;b543b351fe95:38779 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:40:02,629 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32775 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b543b351fe95,35247,1733463601969 2024-12-06T05:40:02,629 DEBUG [RS:1;b543b351fe95:38779 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34453 2024-12-06T05:40:02,629 DEBUG [RS:1;b543b351fe95:38779 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T05:40:02,629 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32775 {}] master.ServerManager(517): Registering regionserver=b543b351fe95,35247,1733463601969 2024-12-06T05:40:02,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T05:40:02,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T05:40:02,631 DEBUG [RS:0;b543b351fe95:35247 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:40:02,631 DEBUG [RS:0;b543b351fe95:35247 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34453 2024-12-06T05:40:02,631 DEBUG [RS:1;b543b351fe95:38779 {}] zookeeper.ZKUtil(111): regionserver:38779-0x100ae0a5c73001c, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b543b351fe95,38779,1733463601993 2024-12-06T05:40:02,631 DEBUG [RS:0;b543b351fe95:35247 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T05:40:02,631 WARN [RS:1;b543b351fe95:38779 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T05:40:02,631 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b543b351fe95,38779,1733463601993] 2024-12-06T05:40:02,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T05:40:02,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T05:40:02,632 INFO [RS:1;b543b351fe95:38779 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:40:02,632 DEBUG [RS:1;b543b351fe95:38779 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,38779,1733463601993 2024-12-06T05:40:02,633 DEBUG [RS:0;b543b351fe95:35247 {}] zookeeper.ZKUtil(111): regionserver:35247-0x100ae0a5c73001b, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b543b351fe95,35247,1733463601969 2024-12-06T05:40:02,633 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b543b351fe95,35247,1733463601969] 2024-12-06T05:40:02,633 WARN [RS:0;b543b351fe95:35247 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T05:40:02,633 INFO [RS:0;b543b351fe95:35247 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:40:02,633 DEBUG [RS:0;b543b351fe95:35247 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,35247,1733463601969 2024-12-06T05:40:02,636 INFO [RS:1;b543b351fe95:38779 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T05:40:02,637 INFO [RS:1;b543b351fe95:38779 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T05:40:02,640 INFO [RS:1;b543b351fe95:38779 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T05:40:02,640 INFO [RS:1;b543b351fe95:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:02,640 INFO [RS:1;b543b351fe95:38779 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T05:40:02,641 INFO [RS:1;b543b351fe95:38779 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T05:40:02,641 INFO [RS:1;b543b351fe95:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:02,641 DEBUG [RS:1;b543b351fe95:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:02,641 DEBUG [RS:1;b543b351fe95:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:02,641 DEBUG [RS:1;b543b351fe95:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:02,641 DEBUG [RS:1;b543b351fe95:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:02,641 DEBUG [RS:1;b543b351fe95:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:02,641 DEBUG [RS:1;b543b351fe95:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0, corePoolSize=2, maxPoolSize=2 2024-12-06T05:40:02,641 DEBUG [RS:1;b543b351fe95:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:02,642 DEBUG [RS:1;b543b351fe95:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:02,642 DEBUG [RS:1;b543b351fe95:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:02,642 DEBUG [RS:1;b543b351fe95:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:02,642 DEBUG [RS:1;b543b351fe95:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:02,642 DEBUG [RS:1;b543b351fe95:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:02,642 DEBUG [RS:1;b543b351fe95:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b543b351fe95:0, corePoolSize=3, maxPoolSize=3 2024-12-06T05:40:02,642 DEBUG [RS:1;b543b351fe95:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b543b351fe95:0, corePoolSize=3, maxPoolSize=3 2024-12-06T05:40:02,642 INFO [RS:0;b543b351fe95:35247 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T05:40:02,646 INFO [RS:0;b543b351fe95:35247 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T05:40:02,646 INFO [RS:1;b543b351fe95:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:02,646 INFO [RS:1;b543b351fe95:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:02,646 INFO [RS:1;b543b351fe95:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:02,646 INFO [RS:1;b543b351fe95:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:02,646 INFO [RS:1;b543b351fe95:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:02,646 INFO [RS:1;b543b351fe95:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,38779,1733463601993-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T05:40:02,646 INFO [RS:0;b543b351fe95:35247 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T05:40:02,646 INFO [RS:0;b543b351fe95:35247 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:02,646 INFO [RS:0;b543b351fe95:35247 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T05:40:02,647 INFO [RS:0;b543b351fe95:35247 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T05:40:02,647 INFO [RS:0;b543b351fe95:35247 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:02,648 DEBUG [RS:0;b543b351fe95:35247 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:02,648 DEBUG [RS:0;b543b351fe95:35247 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:02,648 DEBUG [RS:0;b543b351fe95:35247 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:02,648 DEBUG [RS:0;b543b351fe95:35247 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:02,648 DEBUG [RS:0;b543b351fe95:35247 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:02,648 DEBUG [RS:0;b543b351fe95:35247 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0, corePoolSize=2, maxPoolSize=2 2024-12-06T05:40:02,648 DEBUG [RS:0;b543b351fe95:35247 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:02,648 DEBUG [RS:0;b543b351fe95:35247 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:02,648 DEBUG [RS:0;b543b351fe95:35247 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:02,648 DEBUG [RS:0;b543b351fe95:35247 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:02,648 DEBUG [RS:0;b543b351fe95:35247 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:02,648 DEBUG [RS:0;b543b351fe95:35247 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:02,648 DEBUG [RS:0;b543b351fe95:35247 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b543b351fe95:0, corePoolSize=3, maxPoolSize=3 2024-12-06T05:40:02,648 DEBUG [RS:0;b543b351fe95:35247 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b543b351fe95:0, corePoolSize=3, maxPoolSize=3 2024-12-06T05:40:02,651 INFO [RS:0;b543b351fe95:35247 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:02,651 INFO [RS:0;b543b351fe95:35247 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:02,651 INFO [RS:0;b543b351fe95:35247 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:02,651 INFO [RS:0;b543b351fe95:35247 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:02,651 INFO [RS:0;b543b351fe95:35247 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:02,651 INFO [RS:0;b543b351fe95:35247 {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,35247,1733463601969-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T05:40:02,659 INFO [RS:1;b543b351fe95:38779 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T05:40:02,659 INFO [RS:1;b543b351fe95:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,38779,1733463601993-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:02,659 INFO [RS:1;b543b351fe95:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:02,659 INFO [RS:1;b543b351fe95:38779 {}] regionserver.Replication(171): b543b351fe95,38779,1733463601993 started 2024-12-06T05:40:02,664 INFO [RS:0;b543b351fe95:35247 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T05:40:02,664 INFO [RS:0;b543b351fe95:35247 {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,35247,1733463601969-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:02,664 INFO [RS:0;b543b351fe95:35247 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:02,664 INFO [RS:0;b543b351fe95:35247 {}] regionserver.Replication(171): b543b351fe95,35247,1733463601969 started 2024-12-06T05:40:02,671 INFO [RS:1;b543b351fe95:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:02,671 INFO [RS:1;b543b351fe95:38779 {}] regionserver.HRegionServer(1482): Serving as b543b351fe95,38779,1733463601993, RpcServer on b543b351fe95/172.17.0.2:38779, sessionid=0x100ae0a5c73001c 2024-12-06T05:40:02,672 DEBUG [RS:1;b543b351fe95:38779 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T05:40:02,672 DEBUG [RS:1;b543b351fe95:38779 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b543b351fe95,38779,1733463601993 2024-12-06T05:40:02,672 DEBUG [RS:1;b543b351fe95:38779 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b543b351fe95,38779,1733463601993' 2024-12-06T05:40:02,672 DEBUG [RS:1;b543b351fe95:38779 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T05:40:02,672 DEBUG [RS:1;b543b351fe95:38779 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T05:40:02,672 DEBUG [RS:1;b543b351fe95:38779 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T05:40:02,672 DEBUG [RS:1;b543b351fe95:38779 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T05:40:02,673 DEBUG [RS:1;b543b351fe95:38779 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b543b351fe95,38779,1733463601993 2024-12-06T05:40:02,673 DEBUG [RS:1;b543b351fe95:38779 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b543b351fe95,38779,1733463601993' 2024-12-06T05:40:02,673 DEBUG [RS:1;b543b351fe95:38779 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T05:40:02,673 DEBUG [RS:1;b543b351fe95:38779 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T05:40:02,673 DEBUG [RS:1;b543b351fe95:38779 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T05:40:02,673 INFO [RS:1;b543b351fe95:38779 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T05:40:02,673 INFO [RS:1;b543b351fe95:38779 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T05:40:02,676 INFO [RS:0;b543b351fe95:35247 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:02,677 INFO [RS:0;b543b351fe95:35247 {}] regionserver.HRegionServer(1482): Serving as b543b351fe95,35247,1733463601969, RpcServer on b543b351fe95/172.17.0.2:35247, sessionid=0x100ae0a5c73001b 2024-12-06T05:40:02,677 DEBUG [RS:0;b543b351fe95:35247 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T05:40:02,677 DEBUG [RS:0;b543b351fe95:35247 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b543b351fe95,35247,1733463601969 2024-12-06T05:40:02,677 DEBUG [RS:0;b543b351fe95:35247 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b543b351fe95,35247,1733463601969' 2024-12-06T05:40:02,677 DEBUG [RS:0;b543b351fe95:35247 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T05:40:02,677 DEBUG [RS:0;b543b351fe95:35247 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T05:40:02,678 DEBUG [RS:0;b543b351fe95:35247 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T05:40:02,678 DEBUG [RS:0;b543b351fe95:35247 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T05:40:02,678 DEBUG [RS:0;b543b351fe95:35247 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b543b351fe95,35247,1733463601969 2024-12-06T05:40:02,678 DEBUG [RS:0;b543b351fe95:35247 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b543b351fe95,35247,1733463601969' 2024-12-06T05:40:02,678 DEBUG [RS:0;b543b351fe95:35247 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T05:40:02,678 DEBUG [RS:0;b543b351fe95:35247 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T05:40:02,678 DEBUG [RS:0;b543b351fe95:35247 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T05:40:02,678 INFO [RS:0;b543b351fe95:35247 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T05:40:02,678 INFO [RS:0;b543b351fe95:35247 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T05:40:02,741 WARN [b543b351fe95:32775 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T05:40:02,774 INFO [RS:1;b543b351fe95:38779 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:40:02,777 INFO [RS:1;b543b351fe95:38779 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b543b351fe95%2C38779%2C1733463601993, suffix=, logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,38779,1733463601993, archiveDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/oldWALs, maxLogs=32 2024-12-06T05:40:02,779 INFO [RS:0;b543b351fe95:35247 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:40:02,782 INFO [RS:0;b543b351fe95:35247 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b543b351fe95%2C35247%2C1733463601969, suffix=, logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,35247,1733463601969, archiveDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/oldWALs, maxLogs=32 2024-12-06T05:40:02,794 DEBUG [RS:1;b543b351fe95:38779 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,38779,1733463601993/b543b351fe95%2C38779%2C1733463601993.1733463602778, exclude list is [], retry=0 2024-12-06T05:40:02,796 DEBUG [RS:0;b543b351fe95:35247 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,35247,1733463601969/b543b351fe95%2C35247%2C1733463601969.1733463602783, exclude list is [], retry=0 2024-12-06T05:40:02,797 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-80ddb16c-ec66-4e39-a44c-0a80fa143251,DISK] 2024-12-06T05:40:02,797 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39561,DS-4485eaa6-367b-4084-97e1-b834fb8d8596,DISK] 2024-12-06T05:40:02,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39561,DS-4485eaa6-367b-4084-97e1-b834fb8d8596,DISK] 2024-12-06T05:40:02,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-80ddb16c-ec66-4e39-a44c-0a80fa143251,DISK] 2024-12-06T05:40:02,800 INFO [RS:1;b543b351fe95:38779 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,38779,1733463601993/b543b351fe95%2C38779%2C1733463601993.1733463602778 2024-12-06T05:40:02,800 INFO [RS:0;b543b351fe95:35247 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,35247,1733463601969/b543b351fe95%2C35247%2C1733463601969.1733463602783 2024-12-06T05:40:02,800 DEBUG [RS:1;b543b351fe95:38779 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44855:44855),(127.0.0.1/127.0.0.1:43559:43559)] 2024-12-06T05:40:02,800 DEBUG [RS:0;b543b351fe95:35247 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43559:43559),(127.0.0.1/127.0.0.1:44855:44855)] 2024-12-06T05:40:02,991 DEBUG [b543b351fe95:32775 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-06T05:40:02,991 DEBUG [b543b351fe95:32775 {}] balancer.BalancerClusterState(204): Hosts are {b543b351fe95=0} racks are {/default-rack=0} 2024-12-06T05:40:02,996 DEBUG [b543b351fe95:32775 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-06T05:40:02,996 DEBUG [b543b351fe95:32775 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-06T05:40:02,996 DEBUG [b543b351fe95:32775 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-06T05:40:02,996 DEBUG [b543b351fe95:32775 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-06T05:40:02,996 INFO [b543b351fe95:32775 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-06T05:40:02,996 INFO [b543b351fe95:32775 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-06T05:40:02,996 DEBUG [b543b351fe95:32775 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-06T05:40:02,997 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b543b351fe95,38779,1733463601993 2024-12-06T05:40:02,999 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b543b351fe95,38779,1733463601993, state=OPENING 2024-12-06T05:40:03,001 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-06T05:40:03,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:03,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x100ae0a5c73001b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:03,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x100ae0a5c73001c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:03,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:03,003 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:40:03,003 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:40:03,003 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T05:40:03,003 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:40:03,003 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:40:03,003 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b543b351fe95,38779,1733463601993}] 2024-12-06T05:40:03,139 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T05:40:03,156 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T05:40:03,158 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-23-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53475, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T05:40:03,161 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-06T05:40:03,162 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:40:03,162 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-06T05:40:03,163 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b543b351fe95%2C38779%2C1733463601993.meta, suffix=.meta, logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,38779,1733463601993, archiveDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/oldWALs, maxLogs=32 2024-12-06T05:40:03,176 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,38779,1733463601993/b543b351fe95%2C38779%2C1733463601993.meta.1733463603164.meta, exclude list is [], retry=0 2024-12-06T05:40:03,178 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39561,DS-4485eaa6-367b-4084-97e1-b834fb8d8596,DISK] 2024-12-06T05:40:03,178 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-80ddb16c-ec66-4e39-a44c-0a80fa143251,DISK] 2024-12-06T05:40:03,180 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,38779,1733463601993/b543b351fe95%2C38779%2C1733463601993.meta.1733463603164.meta 2024-12-06T05:40:03,180 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44855:44855),(127.0.0.1/127.0.0.1:43559:43559)] 2024-12-06T05:40:03,180 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:40:03,181 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-06T05:40:03,181 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-06T05:40:03,181 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-06T05:40:03,181 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-06T05:40:03,181 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:40:03,181 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-06T05:40:03,181 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-06T05:40:03,183 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-06T05:40:03,184 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T05:40:03,184 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:40:03,184 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:40:03,184 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-06T05:40:03,185 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T05:40:03,185 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:40:03,186 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:40:03,186 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-06T05:40:03,187 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T05:40:03,187 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:40:03,187 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:40:03,187 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-06T05:40:03,188 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T05:40:03,188 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:40:03,189 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:40:03,189 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T05:40:03,190 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/hbase/meta/1588230740 2024-12-06T05:40:03,191 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/hbase/meta/1588230740 2024-12-06T05:40:03,192 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T05:40:03,192 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T05:40:03,193 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T05:40:03,194 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T05:40:03,195 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64560123, jitterRate=-0.03797920048236847}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T05:40:03,195 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-06T05:40:03,196 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733463603181Writing region info on filesystem at 1733463603181Initializing all the Stores at 1733463603182 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463603182Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463603182Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463603182Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733463603182Cleaning up temporary data from old regions at 1733463603192 (+10 ms)Running coprocessor post-open hooks at 1733463603195 (+3 ms)Region opened successfully at 1733463603196 (+1 ms) 2024-12-06T05:40:03,197 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733463603156 2024-12-06T05:40:03,199 DEBUG [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-06T05:40:03,199 INFO [RS_OPEN_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-06T05:40:03,200 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b543b351fe95,38779,1733463601993 2024-12-06T05:40:03,201 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b543b351fe95,38779,1733463601993, state=OPEN 2024-12-06T05:40:03,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:40:03,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:40:03,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x100ae0a5c73001c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:40:03,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x100ae0a5c73001b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:40:03,202 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b543b351fe95,38779,1733463601993 2024-12-06T05:40:03,202 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:40:03,202 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:40:03,202 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:40:03,202 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:40:03,204 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-06T05:40:03,204 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b543b351fe95,38779,1733463601993 in 199 msec 2024-12-06T05:40:03,205 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-06T05:40:03,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 616 msec 2024-12-06T05:40:03,206 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T05:40:03,206 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-06T05:40:03,208 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T05:40:03,208 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b543b351fe95,38779,1733463601993, seqNum=-1] 2024-12-06T05:40:03,208 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T05:40:03,209 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-23-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52527, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T05:40:03,214 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 673 msec 2024-12-06T05:40:03,215 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733463603215, completionTime=-1 2024-12-06T05:40:03,215 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-06T05:40:03,215 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-06T05:40:03,217 INFO [master/b543b351fe95:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-06T05:40:03,217 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733463663217 2024-12-06T05:40:03,217 INFO [master/b543b351fe95:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733463723217 2024-12-06T05:40:03,217 INFO [master/b543b351fe95:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-12-06T05:40:03,217 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-06T05:40:03,218 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,32775,1733463601942-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:03,218 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,32775,1733463601942-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:03,218 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,32775,1733463601942-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:03,218 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b543b351fe95:32775, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:03,218 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:03,219 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:03,220 DEBUG [master/b543b351fe95:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-06T05:40:03,223 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.198sec 2024-12-06T05:40:03,223 INFO [master/b543b351fe95:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-06T05:40:03,223 INFO [master/b543b351fe95:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-06T05:40:03,223 INFO [master/b543b351fe95:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-06T05:40:03,223 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-06T05:40:03,223 INFO [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-06T05:40:03,223 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,32775,1733463601942-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T05:40:03,223 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,32775,1733463601942-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-06T05:40:03,225 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-06T05:40:03,226 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-06T05:40:03,226 INFO [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,32775,1733463601942-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:03,311 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2f257e98 to 127.0.0.1:58086 with session timeout=1000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5b723c7c 2024-12-06T05:40:03,315 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a7dd01e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T05:40:03,315 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T05:40:03,316 DEBUG [ReadOnlyZKClient-127.0.0.1:58086@0x2f257e98 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b543b351fe95,38779,1733463601993, seqNum=-1] 2024-12-06T05:40:03,317 DEBUG [ReadOnlyZKClient-127.0.0.1:58086@0x2f257e98 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T05:40:03,319 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-23-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55130, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T05:40:03,321 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=b543b351fe95,32775,1733463601942 2024-12-06T05:40:03,337 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b543b351fe95:0 server-side Connection retries=45 2024-12-06T05:40:03,337 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:40:03,337 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T05:40:03,337 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T05:40:03,337 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:40:03,337 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T05:40:03,337 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T05:40:03,337 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T05:40:03,337 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39035 2024-12-06T05:40:03,338 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39035 connecting to ZooKeeper ensemble=127.0.0.1:58086 2024-12-06T05:40:03,339 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:40:03,340 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:40:03,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:390350x0, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:40:03,343 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:39035-0x100ae0a5c73001e, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-06T05:40:03,343 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39035-0x100ae0a5c73001e connected 2024-12-06T05:40:03,343 DEBUG [pool-929-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-06T05:40:03,344 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T05:40:03,344 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T05:40:03,345 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:39035-0x100ae0a5c73001e, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T05:40:03,346 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39035-0x100ae0a5c73001e, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T05:40:03,346 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39035 2024-12-06T05:40:03,346 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39035 2024-12-06T05:40:03,346 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39035 2024-12-06T05:40:03,347 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39035 2024-12-06T05:40:03,347 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39035 2024-12-06T05:40:03,348 INFO [RS:2;b543b351fe95:39035 {}] regionserver.HRegionServer(746): ClusterId : 976c7c09-c307-4348-ab53-202fa67db969 2024-12-06T05:40:03,348 DEBUG [RS:2;b543b351fe95:39035 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T05:40:03,349 DEBUG [RS:2;b543b351fe95:39035 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T05:40:03,349 DEBUG [RS:2;b543b351fe95:39035 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T05:40:03,350 DEBUG [RS:2;b543b351fe95:39035 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T05:40:03,350 DEBUG [RS:2;b543b351fe95:39035 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@308e48f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b543b351fe95/172.17.0.2:0 2024-12-06T05:40:03,360 DEBUG [RS:2;b543b351fe95:39035 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;b543b351fe95:39035 2024-12-06T05:40:03,360 INFO [RS:2;b543b351fe95:39035 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T05:40:03,360 INFO [RS:2;b543b351fe95:39035 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T05:40:03,360 DEBUG [RS:2;b543b351fe95:39035 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T05:40:03,361 INFO [RS:2;b543b351fe95:39035 {}] regionserver.HRegionServer(2659): reportForDuty to master=b543b351fe95,32775,1733463601942 with port=39035, startcode=1733463603336 2024-12-06T05:40:03,361 DEBUG [RS:2;b543b351fe95:39035 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T05:40:03,362 INFO [HMaster-EventLoopGroup-21-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58413, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.11 (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T05:40:03,363 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32775 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b543b351fe95,39035,1733463603336 2024-12-06T05:40:03,363 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32775 {}] master.ServerManager(517): Registering regionserver=b543b351fe95,39035,1733463603336 2024-12-06T05:40:03,364 DEBUG [RS:2;b543b351fe95:39035 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:40:03,364 DEBUG [RS:2;b543b351fe95:39035 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34453 2024-12-06T05:40:03,364 DEBUG [RS:2;b543b351fe95:39035 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T05:40:03,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T05:40:03,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T05:40:03,366 DEBUG [RS:2;b543b351fe95:39035 {}] zookeeper.ZKUtil(111): regionserver:39035-0x100ae0a5c73001e, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b543b351fe95,39035,1733463603336 2024-12-06T05:40:03,366 WARN [RS:2;b543b351fe95:39035 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T05:40:03,366 INFO [RS:2;b543b351fe95:39035 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:40:03,366 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b543b351fe95,39035,1733463603336] 2024-12-06T05:40:03,366 DEBUG [RS:2;b543b351fe95:39035 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,39035,1733463603336 2024-12-06T05:40:03,369 INFO [RS:2;b543b351fe95:39035 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T05:40:03,371 INFO [RS:2;b543b351fe95:39035 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T05:40:03,371 INFO [RS:2;b543b351fe95:39035 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T05:40:03,371 INFO [RS:2;b543b351fe95:39035 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:03,377 INFO [RS:2;b543b351fe95:39035 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T05:40:03,378 INFO [RS:2;b543b351fe95:39035 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T05:40:03,378 INFO [RS:2;b543b351fe95:39035 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:03,378 DEBUG [RS:2;b543b351fe95:39035 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:03,378 DEBUG [RS:2;b543b351fe95:39035 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:03,378 DEBUG [RS:2;b543b351fe95:39035 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:03,378 DEBUG [RS:2;b543b351fe95:39035 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:03,379 DEBUG [RS:2;b543b351fe95:39035 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:03,379 DEBUG [RS:2;b543b351fe95:39035 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b543b351fe95:0, corePoolSize=2, maxPoolSize=2 2024-12-06T05:40:03,379 DEBUG [RS:2;b543b351fe95:39035 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:03,379 DEBUG [RS:2;b543b351fe95:39035 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:03,379 DEBUG [RS:2;b543b351fe95:39035 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:03,379 DEBUG [RS:2;b543b351fe95:39035 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:03,379 DEBUG [RS:2;b543b351fe95:39035 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:03,379 DEBUG [RS:2;b543b351fe95:39035 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b543b351fe95:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:40:03,379 DEBUG [RS:2;b543b351fe95:39035 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b543b351fe95:0, corePoolSize=3, maxPoolSize=3 2024-12-06T05:40:03,379 DEBUG [RS:2;b543b351fe95:39035 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b543b351fe95:0, corePoolSize=3, maxPoolSize=3 2024-12-06T05:40:03,381 INFO [RS:2;b543b351fe95:39035 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:03,381 INFO [RS:2;b543b351fe95:39035 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:03,381 INFO [RS:2;b543b351fe95:39035 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:03,381 INFO [RS:2;b543b351fe95:39035 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:03,381 INFO [RS:2;b543b351fe95:39035 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:03,381 INFO [RS:2;b543b351fe95:39035 {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,39035,1733463603336-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T05:40:03,395 INFO [RS:2;b543b351fe95:39035 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T05:40:03,395 INFO [RS:2;b543b351fe95:39035 {}] hbase.ChoreService(168): Chore ScheduledChore name=b543b351fe95,39035,1733463603336-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:03,395 INFO [RS:2;b543b351fe95:39035 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:03,395 INFO [RS:2;b543b351fe95:39035 {}] regionserver.Replication(171): b543b351fe95,39035,1733463603336 started 2024-12-06T05:40:03,407 INFO [RS:2;b543b351fe95:39035 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:40:03,407 INFO [RS:2;b543b351fe95:39035 {}] regionserver.HRegionServer(1482): Serving as b543b351fe95,39035,1733463603336, RpcServer on b543b351fe95/172.17.0.2:39035, sessionid=0x100ae0a5c73001e 2024-12-06T05:40:03,407 DEBUG [RS:2;b543b351fe95:39035 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T05:40:03,407 DEBUG [RS:2;b543b351fe95:39035 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b543b351fe95,39035,1733463603336 2024-12-06T05:40:03,407 DEBUG [RS:2;b543b351fe95:39035 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b543b351fe95,39035,1733463603336' 2024-12-06T05:40:03,407 DEBUG [RS:2;b543b351fe95:39035 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T05:40:03,407 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T05:40:03,407 DEBUG [RS:2;b543b351fe95:39035 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T05:40:03,408 DEBUG [ReadOnlyZKClient-127.0.0.1:58086@0x2f257e98 {}] client.AsyncConnectionImpl(321): The fetched master address is b543b351fe95,32775,1733463601942 2024-12-06T05:40:03,408 DEBUG [ReadOnlyZKClient-127.0.0.1:58086@0x2f257e98 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2b2047a5 2024-12-06T05:40:03,408 DEBUG [RS:2;b543b351fe95:39035 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T05:40:03,408 DEBUG [RS:2;b543b351fe95:39035 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T05:40:03,408 DEBUG [RS:2;b543b351fe95:39035 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b543b351fe95,39035,1733463603336 2024-12-06T05:40:03,408 DEBUG [ReadOnlyZKClient-127.0.0.1:58086@0x2f257e98 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T05:40:03,408 DEBUG [RS:2;b543b351fe95:39035 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b543b351fe95,39035,1733463603336' 2024-12-06T05:40:03,408 DEBUG [RS:2;b543b351fe95:39035 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T05:40:03,408 DEBUG [RS:2;b543b351fe95:39035 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T05:40:03,408 DEBUG [RS:2;b543b351fe95:39035 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T05:40:03,408 INFO [RS:2;b543b351fe95:39035 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T05:40:03,408 INFO [RS:2;b543b351fe95:39035 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T05:40:03,409 INFO [HMaster-EventLoopGroup-21-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53836, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T05:40:03,410 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32775 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T05:40:03,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32775 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry 2024-12-06T05:40:03,412 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T05:40:03,413 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:40:03,413 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32775 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRegionAssignmentAfterMasterRecoveryDueToZKExpiry" procId is: 4 2024-12-06T05:40:03,414 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T05:40:03,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T05:40:03,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741918_1095 (size=404) 2024-12-06T05:40:03,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741918_1095 (size=404) 2024-12-06T05:40:03,423 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ced409d1a7981960060534bbb519e5d7, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1733463603410.ced409d1a7981960060534bbb519e5d7.', STARTKEY => '', ENDKEY => 'a'}, tableDescriptor='testRegionAssignmentAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:40:03,424 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 23e1669d19d0104a438a531c9f1c918d, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1733463603410.23e1669d19d0104a438a531c9f1c918d.', STARTKEY => 'b', ENDKEY => 'c'}, tableDescriptor='testRegionAssignmentAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:40:03,424 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 16986c34f3a71270f1d68aff722e562a, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1733463603410.16986c34f3a71270f1d68aff722e562a.', STARTKEY => 'a', ENDKEY => 'b'}, tableDescriptor='testRegionAssignmentAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:40:03,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741919_1096 (size=89) 2024-12-06T05:40:03,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741919_1096 (size=89) 2024-12-06T05:40:03,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741920_1097 (size=89) 2024-12-06T05:40:03,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741920_1097 (size=89) 2024-12-06T05:40:03,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741921_1098 (size=88) 2024-12-06T05:40:03,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741921_1098 (size=88) 2024-12-06T05:40:03,433 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1733463603410.ced409d1a7981960060534bbb519e5d7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:40:03,433 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1722): Closing ced409d1a7981960060534bbb519e5d7, disabling compactions & flushes 2024-12-06T05:40:03,433 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1733463603410.ced409d1a7981960060534bbb519e5d7. 2024-12-06T05:40:03,433 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1733463603410.ced409d1a7981960060534bbb519e5d7. 2024-12-06T05:40:03,433 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1733463603410.ced409d1a7981960060534bbb519e5d7. after waiting 0 ms 2024-12-06T05:40:03,433 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1733463603410.ced409d1a7981960060534bbb519e5d7. 2024-12-06T05:40:03,433 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1733463603410.ced409d1a7981960060534bbb519e5d7. 2024-12-06T05:40:03,433 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1676): Region close journal for ced409d1a7981960060534bbb519e5d7: Waiting for close lock at 1733463603433Disabling compacts and flushes for region at 1733463603433Disabling writes for close at 1733463603433Writing region close event to WAL at 1733463603433Closed at 1733463603433 2024-12-06T05:40:03,434 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c0251d107f2e4c28a8d176e8af25b695, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1733463603410.c0251d107f2e4c28a8d176e8af25b695.', STARTKEY => 'c', ENDKEY => 'd'}, tableDescriptor='testRegionAssignmentAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:40:03,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741922_1099 (size=89) 2024-12-06T05:40:03,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741922_1099 (size=89) 2024-12-06T05:40:03,443 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1733463603410.c0251d107f2e4c28a8d176e8af25b695.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:40:03,443 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1722): Closing c0251d107f2e4c28a8d176e8af25b695, disabling compactions & flushes 2024-12-06T05:40:03,443 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1733463603410.c0251d107f2e4c28a8d176e8af25b695. 2024-12-06T05:40:03,443 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1733463603410.c0251d107f2e4c28a8d176e8af25b695. 2024-12-06T05:40:03,443 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1733463603410.c0251d107f2e4c28a8d176e8af25b695. after waiting 0 ms 2024-12-06T05:40:03,443 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1733463603410.c0251d107f2e4c28a8d176e8af25b695. 2024-12-06T05:40:03,443 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1733463603410.c0251d107f2e4c28a8d176e8af25b695. 2024-12-06T05:40:03,443 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1676): Region close journal for c0251d107f2e4c28a8d176e8af25b695: Waiting for close lock at 1733463603443Disabling compacts and flushes for region at 1733463603443Disabling writes for close at 1733463603443Writing region close event to WAL at 1733463603443Closed at 1733463603443 2024-12-06T05:40:03,444 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6b7648f7bc7a5892f02675835e253ce4, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1733463603410.6b7648f7bc7a5892f02675835e253ce4.', STARTKEY => 'd', ENDKEY => 'e'}, tableDescriptor='testRegionAssignmentAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:40:03,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741923_1100 (size=89) 2024-12-06T05:40:03,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741923_1100 (size=89) 2024-12-06T05:40:03,451 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1733463603410.6b7648f7bc7a5892f02675835e253ce4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:40:03,451 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1722): Closing 6b7648f7bc7a5892f02675835e253ce4, disabling compactions & flushes 2024-12-06T05:40:03,451 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-06T05:40:03,451 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1733463603410.6b7648f7bc7a5892f02675835e253ce4. 2024-12-06T05:40:03,452 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1733463603410.6b7648f7bc7a5892f02675835e253ce4. 2024-12-06T05:40:03,452 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1733463603410.6b7648f7bc7a5892f02675835e253ce4. after waiting 0 ms 2024-12-06T05:40:03,452 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1733463603410.6b7648f7bc7a5892f02675835e253ce4. 2024-12-06T05:40:03,452 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1733463603410.6b7648f7bc7a5892f02675835e253ce4. 2024-12-06T05:40:03,452 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6b7648f7bc7a5892f02675835e253ce4: Waiting for close lock at 1733463603451Disabling compacts and flushes for region at 1733463603451Disabling writes for close at 1733463603452 (+1 ms)Writing region close event to WAL at 1733463603452Closed at 1733463603452 2024-12-06T05:40:03,452 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5bfbbde54676defb425c75f2d409c104, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1733463603410.5bfbbde54676defb425c75f2d409c104.', STARTKEY => 'e', ENDKEY => 'f'}, tableDescriptor='testRegionAssignmentAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:40:03,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741924_1101 (size=89) 2024-12-06T05:40:03,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741924_1101 (size=89) 2024-12-06T05:40:03,462 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1733463603410.5bfbbde54676defb425c75f2d409c104.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:40:03,462 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1722): Closing 5bfbbde54676defb425c75f2d409c104, disabling compactions & flushes 2024-12-06T05:40:03,462 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1733463603410.5bfbbde54676defb425c75f2d409c104. 2024-12-06T05:40:03,462 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1733463603410.5bfbbde54676defb425c75f2d409c104. 2024-12-06T05:40:03,462 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1733463603410.5bfbbde54676defb425c75f2d409c104. after waiting 0 ms 2024-12-06T05:40:03,462 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1733463603410.5bfbbde54676defb425c75f2d409c104. 2024-12-06T05:40:03,462 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1733463603410.5bfbbde54676defb425c75f2d409c104. 2024-12-06T05:40:03,462 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5bfbbde54676defb425c75f2d409c104: Waiting for close lock at 1733463603462Disabling compacts and flushes for region at 1733463603462Disabling writes for close at 1733463603462Writing region close event to WAL at 1733463603462Closed at 1733463603462 2024-12-06T05:40:03,463 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b20ebfffac3488c42f88544a5529f43f, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1733463603410.b20ebfffac3488c42f88544a5529f43f.', STARTKEY => 'f', ENDKEY => 'g'}, tableDescriptor='testRegionAssignmentAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:40:03,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741925_1102 (size=89) 2024-12-06T05:40:03,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741925_1102 (size=89) 2024-12-06T05:40:03,470 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1733463603410.b20ebfffac3488c42f88544a5529f43f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:40:03,470 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1722): Closing b20ebfffac3488c42f88544a5529f43f, disabling compactions & flushes 2024-12-06T05:40:03,470 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1733463603410.b20ebfffac3488c42f88544a5529f43f. 2024-12-06T05:40:03,470 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1733463603410.b20ebfffac3488c42f88544a5529f43f. 2024-12-06T05:40:03,470 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1733463603410.b20ebfffac3488c42f88544a5529f43f. after waiting 0 ms 2024-12-06T05:40:03,470 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1733463603410.b20ebfffac3488c42f88544a5529f43f. 2024-12-06T05:40:03,470 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1733463603410.b20ebfffac3488c42f88544a5529f43f. 2024-12-06T05:40:03,470 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1676): Region close journal for b20ebfffac3488c42f88544a5529f43f: Waiting for close lock at 1733463603470Disabling compacts and flushes for region at 1733463603470Disabling writes for close at 1733463603470Writing region close event to WAL at 1733463603470Closed at 1733463603470 2024-12-06T05:40:03,471 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8b92523d97289af37e0b4eb02850f93c, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1733463603410.8b92523d97289af37e0b4eb02850f93c.', STARTKEY => 'g', ENDKEY => 'h'}, tableDescriptor='testRegionAssignmentAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:40:03,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741926_1103 (size=89) 2024-12-06T05:40:03,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741926_1103 (size=89) 2024-12-06T05:40:03,509 INFO [RS:2;b543b351fe95:39035 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:40:03,512 INFO [RS:2;b543b351fe95:39035 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b543b351fe95%2C39035%2C1733463603336, suffix=, logDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,39035,1733463603336, archiveDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/oldWALs, maxLogs=32 2024-12-06T05:40:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T05:40:03,527 DEBUG [RS:2;b543b351fe95:39035 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,39035,1733463603336/b543b351fe95%2C39035%2C1733463603336.1733463603513, exclude list is [], retry=0 2024-12-06T05:40:03,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-80ddb16c-ec66-4e39-a44c-0a80fa143251,DISK] 2024-12-06T05:40:03,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39561,DS-4485eaa6-367b-4084-97e1-b834fb8d8596,DISK] 2024-12-06T05:40:03,532 INFO [RS:2;b543b351fe95:39035 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,39035,1733463603336/b543b351fe95%2C39035%2C1733463603336.1733463603513 2024-12-06T05:40:03,532 DEBUG [RS:2;b543b351fe95:39035 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44855:44855),(127.0.0.1/127.0.0.1:43559:43559)] 2024-12-06T05:40:03,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T05:40:03,832 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1733463603410.16986c34f3a71270f1d68aff722e562a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:40:03,832 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1722): Closing 16986c34f3a71270f1d68aff722e562a, disabling compactions & flushes 2024-12-06T05:40:03,832 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1733463603410.16986c34f3a71270f1d68aff722e562a. 2024-12-06T05:40:03,832 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1733463603410.16986c34f3a71270f1d68aff722e562a. 2024-12-06T05:40:03,832 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1733463603410.16986c34f3a71270f1d68aff722e562a. after waiting 0 ms 2024-12-06T05:40:03,832 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1733463603410.23e1669d19d0104a438a531c9f1c918d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:40:03,832 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1733463603410.16986c34f3a71270f1d68aff722e562a. 2024-12-06T05:40:03,832 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1733463603410.16986c34f3a71270f1d68aff722e562a. 2024-12-06T05:40:03,832 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1722): Closing 23e1669d19d0104a438a531c9f1c918d, disabling compactions & flushes 2024-12-06T05:40:03,832 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1676): Region close journal for 16986c34f3a71270f1d68aff722e562a: Waiting for close lock at 1733463603832Disabling compacts and flushes for region at 1733463603832Disabling writes for close at 1733463603832Writing region close event to WAL at 1733463603832Closed at 1733463603832 2024-12-06T05:40:03,832 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1733463603410.23e1669d19d0104a438a531c9f1c918d. 2024-12-06T05:40:03,833 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1733463603410.23e1669d19d0104a438a531c9f1c918d. 2024-12-06T05:40:03,833 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1733463603410.23e1669d19d0104a438a531c9f1c918d. after waiting 0 ms 2024-12-06T05:40:03,833 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1733463603410.23e1669d19d0104a438a531c9f1c918d. 2024-12-06T05:40:03,833 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => a8d0bc8f1ee440609a75868892f81f67, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1733463603410.a8d0bc8f1ee440609a75868892f81f67.', STARTKEY => 'h', ENDKEY => 'i'}, tableDescriptor='testRegionAssignmentAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:40:03,833 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1733463603410.23e1669d19d0104a438a531c9f1c918d. 2024-12-06T05:40:03,833 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1676): Region close journal for 23e1669d19d0104a438a531c9f1c918d: Waiting for close lock at 1733463603832Disabling compacts and flushes for region at 1733463603832Disabling writes for close at 1733463603833 (+1 ms)Writing region close event to WAL at 1733463603833Closed at 1733463603833 2024-12-06T05:40:03,834 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 73b074a80469c17ad4213657d37d496f, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1733463603410.73b074a80469c17ad4213657d37d496f.', STARTKEY => 'i', ENDKEY => 'j'}, tableDescriptor='testRegionAssignmentAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:40:03,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741929_1106 (size=89) 2024-12-06T05:40:03,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741928_1105 (size=89) 2024-12-06T05:40:03,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741928_1105 (size=89) 2024-12-06T05:40:03,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741929_1106 (size=89) 2024-12-06T05:40:03,845 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1733463603410.73b074a80469c17ad4213657d37d496f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:40:03,845 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1733463603410.a8d0bc8f1ee440609a75868892f81f67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:40:03,845 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1722): Closing a8d0bc8f1ee440609a75868892f81f67, disabling compactions & flushes 2024-12-06T05:40:03,845 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1722): Closing 73b074a80469c17ad4213657d37d496f, disabling compactions & flushes 2024-12-06T05:40:03,845 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1733463603410.a8d0bc8f1ee440609a75868892f81f67. 2024-12-06T05:40:03,845 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1733463603410.73b074a80469c17ad4213657d37d496f. 2024-12-06T05:40:03,846 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1733463603410.a8d0bc8f1ee440609a75868892f81f67. 2024-12-06T05:40:03,846 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1733463603410.73b074a80469c17ad4213657d37d496f. 2024-12-06T05:40:03,846 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1733463603410.a8d0bc8f1ee440609a75868892f81f67. after waiting 0 ms 2024-12-06T05:40:03,846 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1733463603410.73b074a80469c17ad4213657d37d496f. after waiting 0 ms 2024-12-06T05:40:03,846 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1733463603410.a8d0bc8f1ee440609a75868892f81f67. 2024-12-06T05:40:03,846 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1733463603410.73b074a80469c17ad4213657d37d496f. 2024-12-06T05:40:03,846 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1733463603410.a8d0bc8f1ee440609a75868892f81f67. 2024-12-06T05:40:03,846 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1733463603410.73b074a80469c17ad4213657d37d496f. 2024-12-06T05:40:03,846 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-1 {}] regionserver.HRegion(1676): Region close journal for a8d0bc8f1ee440609a75868892f81f67: Waiting for close lock at 1733463603845Disabling compacts and flushes for region at 1733463603845Disabling writes for close at 1733463603846 (+1 ms)Writing region close event to WAL at 1733463603846Closed at 1733463603846 2024-12-06T05:40:03,846 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1676): Region close journal for 73b074a80469c17ad4213657d37d496f: Waiting for close lock at 1733463603845Disabling compacts and flushes for region at 1733463603845Disabling writes for close at 1733463603846 (+1 ms)Writing region close event to WAL at 1733463603846Closed at 1733463603846 2024-12-06T05:40:03,846 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => cc3edcebfba16936afa2293e025fd871, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1733463603410.cc3edcebfba16936afa2293e025fd871.', STARTKEY => 'j', ENDKEY => ''}, tableDescriptor='testRegionAssignmentAfterMasterRecoveryDueToZKExpiry', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279 2024-12-06T05:40:03,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741930_1107 (size=88) 2024-12-06T05:40:03,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741930_1107 (size=88) 2024-12-06T05:40:03,855 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1733463603410.cc3edcebfba16936afa2293e025fd871.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:40:03,855 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1722): Closing cc3edcebfba16936afa2293e025fd871, disabling compactions & flushes 2024-12-06T05:40:03,855 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1733463603410.cc3edcebfba16936afa2293e025fd871. 2024-12-06T05:40:03,855 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1733463603410.cc3edcebfba16936afa2293e025fd871. 2024-12-06T05:40:03,855 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1733463603410.cc3edcebfba16936afa2293e025fd871. after waiting 0 ms 2024-12-06T05:40:03,855 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1733463603410.cc3edcebfba16936afa2293e025fd871. 2024-12-06T05:40:03,855 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1733463603410.cc3edcebfba16936afa2293e025fd871. 2024-12-06T05:40:03,855 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-2 {}] regionserver.HRegion(1676): Region close journal for cc3edcebfba16936afa2293e025fd871: Waiting for close lock at 1733463603855Disabling compacts and flushes for region at 1733463603855Disabling writes for close at 1733463603855Writing region close event to WAL at 1733463603855Closed at 1733463603855 2024-12-06T05:40:03,878 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1733463603410.8b92523d97289af37e0b4eb02850f93c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:40:03,878 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1722): Closing 8b92523d97289af37e0b4eb02850f93c, disabling compactions & flushes 2024-12-06T05:40:03,878 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1733463603410.8b92523d97289af37e0b4eb02850f93c. 2024-12-06T05:40:03,878 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1733463603410.8b92523d97289af37e0b4eb02850f93c. 2024-12-06T05:40:03,878 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1733463603410.8b92523d97289af37e0b4eb02850f93c. after waiting 0 ms 2024-12-06T05:40:03,878 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1733463603410.8b92523d97289af37e0b4eb02850f93c. 2024-12-06T05:40:03,878 INFO [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1733463603410.8b92523d97289af37e0b4eb02850f93c. 2024-12-06T05:40:03,878 DEBUG [RegionOpenAndInit-testRegionAssignmentAfterMasterRecoveryDueToZKExpiry-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8b92523d97289af37e0b4eb02850f93c: Waiting for close lock at 1733463603878Disabling compacts and flushes for region at 1733463603878Disabling writes for close at 1733463603878Writing region close event to WAL at 1733463603878Closed at 1733463603878 2024-12-06T05:40:03,880 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T05:40:03,880 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1733463603410.ced409d1a7981960060534bbb519e5d7.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1733463603880"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733463603880"}]},"ts":"1733463603880"} 2024-12-06T05:40:03,880 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1733463603410.c0251d107f2e4c28a8d176e8af25b695.","families":{"info":[{"qualifier":"regioninfo","vlen":88,"tag":[],"timestamp":"1733463603880"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733463603880"}]},"ts":"1733463603880"} 2024-12-06T05:40:03,880 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1733463603410.6b7648f7bc7a5892f02675835e253ce4.","families":{"info":[{"qualifier":"regioninfo","vlen":88,"tag":[],"timestamp":"1733463603880"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733463603880"}]},"ts":"1733463603880"} 2024-12-06T05:40:03,881 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1733463603410.5bfbbde54676defb425c75f2d409c104.","families":{"info":[{"qualifier":"regioninfo","vlen":88,"tag":[],"timestamp":"1733463603880"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733463603880"}]},"ts":"1733463603880"} 2024-12-06T05:40:03,881 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1733463603410.b20ebfffac3488c42f88544a5529f43f.","families":{"info":[{"qualifier":"regioninfo","vlen":88,"tag":[],"timestamp":"1733463603880"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733463603880"}]},"ts":"1733463603880"} 2024-12-06T05:40:03,881 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1733463603410.16986c34f3a71270f1d68aff722e562a.","families":{"info":[{"qualifier":"regioninfo","vlen":88,"tag":[],"timestamp":"1733463603880"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733463603880"}]},"ts":"1733463603880"} 2024-12-06T05:40:03,881 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1733463603410.23e1669d19d0104a438a531c9f1c918d.","families":{"info":[{"qualifier":"regioninfo","vlen":88,"tag":[],"timestamp":"1733463603880"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733463603880"}]},"ts":"1733463603880"} 2024-12-06T05:40:03,881 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1733463603410.a8d0bc8f1ee440609a75868892f81f67.","families":{"info":[{"qualifier":"regioninfo","vlen":88,"tag":[],"timestamp":"1733463603880"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733463603880"}]},"ts":"1733463603880"} 2024-12-06T05:40:03,881 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1733463603410.73b074a80469c17ad4213657d37d496f.","families":{"info":[{"qualifier":"regioninfo","vlen":88,"tag":[],"timestamp":"1733463603880"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733463603880"}]},"ts":"1733463603880"} 2024-12-06T05:40:03,881 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1733463603410.cc3edcebfba16936afa2293e025fd871.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1733463603880"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733463603880"}]},"ts":"1733463603880"} 2024-12-06T05:40:03,881 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1733463603410.8b92523d97289af37e0b4eb02850f93c.","families":{"info":[{"qualifier":"regioninfo","vlen":88,"tag":[],"timestamp":"1733463603880"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733463603880"}]},"ts":"1733463603880"} 2024-12-06T05:40:03,886 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 11 regions to meta. 2024-12-06T05:40:03,887 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T05:40:03,887 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRegionAssignmentAfterMasterRecoveryDueToZKExpiry","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733463603887"}]},"ts":"1733463603887"} 2024-12-06T05:40:03,889 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, state=ENABLING in hbase:meta 2024-12-06T05:40:03,890 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {b543b351fe95=0} racks are {/default-rack=0} 2024-12-06T05:40:03,895 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-06T05:40:03,895 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-06T05:40:03,895 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-06T05:40:03,895 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-06T05:40:03,895 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-06T05:40:03,895 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-06T05:40:03,895 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-06T05:40:03,895 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-06T05:40:03,895 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-06T05:40:03,895 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-06T05:40:03,896 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=ced409d1a7981960060534bbb519e5d7, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=16986c34f3a71270f1d68aff722e562a, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=23e1669d19d0104a438a531c9f1c918d, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=c0251d107f2e4c28a8d176e8af25b695, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=6b7648f7bc7a5892f02675835e253ce4, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=5bfbbde54676defb425c75f2d409c104, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=b20ebfffac3488c42f88544a5529f43f, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=8b92523d97289af37e0b4eb02850f93c, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=a8d0bc8f1ee440609a75868892f81f67, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=73b074a80469c17ad4213657d37d496f, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=cc3edcebfba16936afa2293e025fd871, ASSIGN}] 2024-12-06T05:40:03,897 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=ced409d1a7981960060534bbb519e5d7, ASSIGN 2024-12-06T05:40:03,897 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=5bfbbde54676defb425c75f2d409c104, ASSIGN 2024-12-06T05:40:03,898 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=16986c34f3a71270f1d68aff722e562a, ASSIGN 2024-12-06T05:40:03,898 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=6b7648f7bc7a5892f02675835e253ce4, ASSIGN 2024-12-06T05:40:03,898 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=ced409d1a7981960060534bbb519e5d7, ASSIGN; state=OFFLINE, location=b543b351fe95,38779,1733463601993; forceNewPlan=false, retain=false 2024-12-06T05:40:03,898 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=5bfbbde54676defb425c75f2d409c104, ASSIGN; state=OFFLINE, location=b543b351fe95,38779,1733463601993; forceNewPlan=false, retain=false 2024-12-06T05:40:03,898 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=16986c34f3a71270f1d68aff722e562a, ASSIGN; state=OFFLINE, location=b543b351fe95,35247,1733463601969; forceNewPlan=false, retain=false 2024-12-06T05:40:03,898 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=6b7648f7bc7a5892f02675835e253ce4, ASSIGN; state=OFFLINE, location=b543b351fe95,35247,1733463601969; forceNewPlan=false, retain=false 2024-12-06T05:40:03,901 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=a8d0bc8f1ee440609a75868892f81f67, ASSIGN 2024-12-06T05:40:03,901 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=8b92523d97289af37e0b4eb02850f93c, ASSIGN 2024-12-06T05:40:03,901 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=73b074a80469c17ad4213657d37d496f, ASSIGN 2024-12-06T05:40:03,901 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=cc3edcebfba16936afa2293e025fd871, ASSIGN 2024-12-06T05:40:03,901 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=b20ebfffac3488c42f88544a5529f43f, ASSIGN 2024-12-06T05:40:03,902 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=cc3edcebfba16936afa2293e025fd871, ASSIGN; state=OFFLINE, location=b543b351fe95,38779,1733463601993; forceNewPlan=false, retain=false 2024-12-06T05:40:03,902 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=a8d0bc8f1ee440609a75868892f81f67, ASSIGN; state=OFFLINE, location=b543b351fe95,39035,1733463603336; forceNewPlan=false, retain=false 2024-12-06T05:40:03,902 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=73b074a80469c17ad4213657d37d496f, ASSIGN; state=OFFLINE, location=b543b351fe95,35247,1733463601969; forceNewPlan=false, retain=false 2024-12-06T05:40:03,902 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=8b92523d97289af37e0b4eb02850f93c, ASSIGN; state=OFFLINE, location=b543b351fe95,39035,1733463603336; forceNewPlan=false, retain=false 2024-12-06T05:40:03,902 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=b20ebfffac3488c42f88544a5529f43f, ASSIGN; state=OFFLINE, location=b543b351fe95,39035,1733463603336; forceNewPlan=false, retain=false 2024-12-06T05:40:03,905 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=23e1669d19d0104a438a531c9f1c918d, ASSIGN 2024-12-06T05:40:03,905 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=c0251d107f2e4c28a8d176e8af25b695, ASSIGN 2024-12-06T05:40:03,906 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=23e1669d19d0104a438a531c9f1c918d, ASSIGN; state=OFFLINE, location=b543b351fe95,38779,1733463601993; forceNewPlan=false, retain=false 2024-12-06T05:40:03,906 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=c0251d107f2e4c28a8d176e8af25b695, ASSIGN; state=OFFLINE, location=b543b351fe95,39035,1733463603336; forceNewPlan=false, retain=false 2024-12-06T05:40:04,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T05:40:04,049 INFO [b543b351fe95:32775 {}] balancer.BaseLoadBalancer(388): Reassigned 11 regions. 11 retained the pre-restart assignment. 2024-12-06T05:40:04,049 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=a8d0bc8f1ee440609a75868892f81f67, regionState=OPENING, regionLocation=b543b351fe95,39035,1733463603336 2024-12-06T05:40:04,049 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=c0251d107f2e4c28a8d176e8af25b695, regionState=OPENING, regionLocation=b543b351fe95,39035,1733463603336 2024-12-06T05:40:04,049 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=cc3edcebfba16936afa2293e025fd871, regionState=OPENING, regionLocation=b543b351fe95,38779,1733463601993 2024-12-06T05:40:04,049 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=8b92523d97289af37e0b4eb02850f93c, regionState=OPENING, regionLocation=b543b351fe95,39035,1733463603336 2024-12-06T05:40:04,049 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=b20ebfffac3488c42f88544a5529f43f, regionState=OPENING, regionLocation=b543b351fe95,39035,1733463603336 2024-12-06T05:40:04,050 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=5bfbbde54676defb425c75f2d409c104, regionState=OPENING, regionLocation=b543b351fe95,38779,1733463601993 2024-12-06T05:40:04,050 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ced409d1a7981960060534bbb519e5d7, regionState=OPENING, regionLocation=b543b351fe95,38779,1733463601993 2024-12-06T05:40:04,050 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=23e1669d19d0104a438a531c9f1c918d, regionState=OPENING, regionLocation=b543b351fe95,38779,1733463601993 2024-12-06T05:40:04,050 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=73b074a80469c17ad4213657d37d496f, regionState=OPENING, regionLocation=b543b351fe95,35247,1733463601969 2024-12-06T05:40:04,050 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=6b7648f7bc7a5892f02675835e253ce4, regionState=OPENING, regionLocation=b543b351fe95,35247,1733463601969 2024-12-06T05:40:04,050 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=16986c34f3a71270f1d68aff722e562a, regionState=OPENING, regionLocation=b543b351fe95,35247,1733463601969 2024-12-06T05:40:04,052 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=a8d0bc8f1ee440609a75868892f81f67, ASSIGN because future has completed 2024-12-06T05:40:04,053 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure a8d0bc8f1ee440609a75868892f81f67, server=b543b351fe95,39035,1733463603336}] 2024-12-06T05:40:04,054 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=8b92523d97289af37e0b4eb02850f93c, ASSIGN because future has completed 2024-12-06T05:40:04,054 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b92523d97289af37e0b4eb02850f93c, server=b543b351fe95,39035,1733463603336}] 2024-12-06T05:40:04,055 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=b20ebfffac3488c42f88544a5529f43f, ASSIGN because future has completed 2024-12-06T05:40:04,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=c0251d107f2e4c28a8d176e8af25b695, ASSIGN because future has completed 2024-12-06T05:40:04,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=cc3edcebfba16936afa2293e025fd871, ASSIGN because future has completed 2024-12-06T05:40:04,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=6b7648f7bc7a5892f02675835e253ce4, ASSIGN because future has completed 2024-12-06T05:40:04,058 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure cc3edcebfba16936afa2293e025fd871, server=b543b351fe95,38779,1733463601993}] 2024-12-06T05:40:04,058 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure b20ebfffac3488c42f88544a5529f43f, server=b543b351fe95,39035,1733463603336}] 2024-12-06T05:40:04,061 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0251d107f2e4c28a8d176e8af25b695, server=b543b351fe95,39035,1733463603336}] 2024-12-06T05:40:04,066 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6b7648f7bc7a5892f02675835e253ce4, server=b543b351fe95,35247,1733463601969}] 2024-12-06T05:40:04,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=5bfbbde54676defb425c75f2d409c104, ASSIGN because future has completed 2024-12-06T05:40:04,068 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bfbbde54676defb425c75f2d409c104, server=b543b351fe95,38779,1733463601993}] 2024-12-06T05:40:04,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=ced409d1a7981960060534bbb519e5d7, ASSIGN because future has completed 2024-12-06T05:40:04,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=23e1669d19d0104a438a531c9f1c918d, ASSIGN because future has completed 2024-12-06T05:40:04,070 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ced409d1a7981960060534bbb519e5d7, server=b543b351fe95,38779,1733463601993}] 2024-12-06T05:40:04,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23e1669d19d0104a438a531c9f1c918d, server=b543b351fe95,38779,1733463601993}] 2024-12-06T05:40:04,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=73b074a80469c17ad4213657d37d496f, ASSIGN because future has completed 2024-12-06T05:40:04,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=16986c34f3a71270f1d68aff722e562a, ASSIGN because future has completed 2024-12-06T05:40:04,072 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 73b074a80469c17ad4213657d37d496f, server=b543b351fe95,35247,1733463601969}] 2024-12-06T05:40:04,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 16986c34f3a71270f1d68aff722e562a, server=b543b351fe95,35247,1733463601969}] 2024-12-06T05:40:04,206 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T05:40:04,208 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-24-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51977, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T05:40:04,216 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(132): Open testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1733463603410.a8d0bc8f1ee440609a75868892f81f67. 2024-12-06T05:40:04,217 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7752): Opening region: {ENCODED => a8d0bc8f1ee440609a75868892f81f67, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1733463603410.a8d0bc8f1ee440609a75868892f81f67.', STARTKEY => 'h', ENDKEY => 'i'} 2024-12-06T05:40:04,217 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRegionAssignmentAfterMasterRecoveryDueToZKExpiry a8d0bc8f1ee440609a75868892f81f67 2024-12-06T05:40:04,217 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1733463603410.a8d0bc8f1ee440609a75868892f81f67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:40:04,217 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7794): checking encryption for a8d0bc8f1ee440609a75868892f81f67 2024-12-06T05:40:04,218 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7797): checking classloading for a8d0bc8f1ee440609a75868892f81f67 2024-12-06T05:40:04,219 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T05:40:04,219 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(132): Open testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1733463603410.ced409d1a7981960060534bbb519e5d7. 2024-12-06T05:40:04,219 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7752): Opening region: {ENCODED => ced409d1a7981960060534bbb519e5d7, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1733463603410.ced409d1a7981960060534bbb519e5d7.', STARTKEY => '', ENDKEY => 'a'} 2024-12-06T05:40:04,219 INFO [StoreOpener-a8d0bc8f1ee440609a75868892f81f67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region a8d0bc8f1ee440609a75868892f81f67 2024-12-06T05:40:04,219 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRegionAssignmentAfterMasterRecoveryDueToZKExpiry ced409d1a7981960060534bbb519e5d7 2024-12-06T05:40:04,219 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1733463603410.ced409d1a7981960060534bbb519e5d7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:40:04,219 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7794): checking encryption for ced409d1a7981960060534bbb519e5d7 2024-12-06T05:40:04,219 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7797): checking classloading for ced409d1a7981960060534bbb519e5d7 2024-12-06T05:40:04,220 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-22-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58499, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T05:40:04,221 INFO [StoreOpener-a8d0bc8f1ee440609a75868892f81f67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a8d0bc8f1ee440609a75868892f81f67 columnFamilyName info 2024-12-06T05:40:04,221 DEBUG [StoreOpener-a8d0bc8f1ee440609a75868892f81f67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:40:04,221 INFO [StoreOpener-a8d0bc8f1ee440609a75868892f81f67-1 {}] regionserver.HStore(327): Store=a8d0bc8f1ee440609a75868892f81f67/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:40:04,221 INFO [StoreOpener-ced409d1a7981960060534bbb519e5d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region ced409d1a7981960060534bbb519e5d7 2024-12-06T05:40:04,221 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1038): replaying wal for a8d0bc8f1ee440609a75868892f81f67 2024-12-06T05:40:04,222 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/a8d0bc8f1ee440609a75868892f81f67 2024-12-06T05:40:04,222 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/a8d0bc8f1ee440609a75868892f81f67 2024-12-06T05:40:04,223 INFO [StoreOpener-ced409d1a7981960060534bbb519e5d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ced409d1a7981960060534bbb519e5d7 columnFamilyName info 2024-12-06T05:40:04,223 DEBUG [StoreOpener-ced409d1a7981960060534bbb519e5d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:40:04,223 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1048): stopping wal replay for a8d0bc8f1ee440609a75868892f81f67 2024-12-06T05:40:04,223 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1060): Cleaning up temporary data for a8d0bc8f1ee440609a75868892f81f67 2024-12-06T05:40:04,223 INFO [StoreOpener-ced409d1a7981960060534bbb519e5d7-1 {}] regionserver.HStore(327): Store=ced409d1a7981960060534bbb519e5d7/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:40:04,223 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1038): replaying wal for ced409d1a7981960060534bbb519e5d7 2024-12-06T05:40:04,224 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/ced409d1a7981960060534bbb519e5d7 2024-12-06T05:40:04,224 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(132): Open testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1733463603410.16986c34f3a71270f1d68aff722e562a. 2024-12-06T05:40:04,224 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7752): Opening region: {ENCODED => 16986c34f3a71270f1d68aff722e562a, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1733463603410.16986c34f3a71270f1d68aff722e562a.', STARTKEY => 'a', ENDKEY => 'b'} 2024-12-06T05:40:04,224 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/ced409d1a7981960060534bbb519e5d7 2024-12-06T05:40:04,225 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRegionAssignmentAfterMasterRecoveryDueToZKExpiry 16986c34f3a71270f1d68aff722e562a 2024-12-06T05:40:04,225 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1733463603410.16986c34f3a71270f1d68aff722e562a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:40:04,225 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1048): stopping wal replay for ced409d1a7981960060534bbb519e5d7 2024-12-06T05:40:04,225 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1060): Cleaning up temporary data for ced409d1a7981960060534bbb519e5d7 2024-12-06T05:40:04,225 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1093): writing seq id for a8d0bc8f1ee440609a75868892f81f67 2024-12-06T05:40:04,225 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7794): checking encryption for 16986c34f3a71270f1d68aff722e562a 2024-12-06T05:40:04,225 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7797): checking classloading for 16986c34f3a71270f1d68aff722e562a 2024-12-06T05:40:04,226 INFO [StoreOpener-16986c34f3a71270f1d68aff722e562a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 16986c34f3a71270f1d68aff722e562a 2024-12-06T05:40:04,227 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1093): writing seq id for ced409d1a7981960060534bbb519e5d7 2024-12-06T05:40:04,227 INFO [StoreOpener-16986c34f3a71270f1d68aff722e562a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 16986c34f3a71270f1d68aff722e562a columnFamilyName info 2024-12-06T05:40:04,227 DEBUG [StoreOpener-16986c34f3a71270f1d68aff722e562a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:40:04,228 INFO [StoreOpener-16986c34f3a71270f1d68aff722e562a-1 {}] regionserver.HStore(327): Store=16986c34f3a71270f1d68aff722e562a/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:40:04,228 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1038): replaying wal for 16986c34f3a71270f1d68aff722e562a 2024-12-06T05:40:04,229 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/16986c34f3a71270f1d68aff722e562a 2024-12-06T05:40:04,229 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/16986c34f3a71270f1d68aff722e562a 2024-12-06T05:40:04,229 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1048): stopping wal replay for 16986c34f3a71270f1d68aff722e562a 2024-12-06T05:40:04,229 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1060): Cleaning up temporary data for 16986c34f3a71270f1d68aff722e562a 2024-12-06T05:40:04,231 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1093): writing seq id for 16986c34f3a71270f1d68aff722e562a 2024-12-06T05:40:04,234 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/a8d0bc8f1ee440609a75868892f81f67/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:40:04,234 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/16986c34f3a71270f1d68aff722e562a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:40:04,234 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/ced409d1a7981960060534bbb519e5d7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:40:04,234 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1114): Opened ced409d1a7981960060534bbb519e5d7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62086070, jitterRate=-0.07484546303749084}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:40:04,234 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1114): Opened 16986c34f3a71270f1d68aff722e562a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58720631, jitterRate=-0.12499441206455231}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:40:04,234 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1114): Opened a8d0bc8f1ee440609a75868892f81f67; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72196973, jitterRate=0.075818732380867}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:40:04,234 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ced409d1a7981960060534bbb519e5d7 2024-12-06T05:40:04,234 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 16986c34f3a71270f1d68aff722e562a 2024-12-06T05:40:04,234 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a8d0bc8f1ee440609a75868892f81f67 2024-12-06T05:40:04,235 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1006): Region open journal for ced409d1a7981960060534bbb519e5d7: Running coprocessor pre-open hook at 1733463604220Writing region info on filesystem at 1733463604220Initializing all the Stores at 1733463604220Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463604221 (+1 ms)Cleaning up temporary data from old regions at 1733463604225 (+4 ms)Running coprocessor post-open hooks at 1733463604234 (+9 ms)Region opened successfully at 1733463604235 (+1 ms) 2024-12-06T05:40:04,235 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1006): Region open journal for 16986c34f3a71270f1d68aff722e562a: Running coprocessor pre-open hook at 1733463604225Writing region info on filesystem at 1733463604225Initializing all the Stores at 1733463604226 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463604226Cleaning up temporary data from old regions at 1733463604229 (+3 ms)Running coprocessor post-open hooks at 1733463604234 (+5 ms)Region opened successfully at 1733463604235 (+1 ms) 2024-12-06T05:40:04,235 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1006): Region open journal for a8d0bc8f1ee440609a75868892f81f67: Running coprocessor pre-open hook at 1733463604218Writing region info on filesystem at 1733463604218Initializing all the Stores at 1733463604219 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463604219Cleaning up temporary data from old regions at 1733463604223 (+4 ms)Running coprocessor post-open hooks at 1733463604234 (+11 ms)Region opened successfully at 1733463604235 (+1 ms) 2024-12-06T05:40:04,236 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2236): Post open deploy tasks for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1733463603410.16986c34f3a71270f1d68aff722e562a., pid=26, masterSystemTime=1733463604218 2024-12-06T05:40:04,236 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2236): Post open deploy tasks for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1733463603410.ced409d1a7981960060534bbb519e5d7., pid=23, masterSystemTime=1733463604213 2024-12-06T05:40:04,236 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2236): Post open deploy tasks for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1733463603410.a8d0bc8f1ee440609a75868892f81f67., pid=16, masterSystemTime=1733463604206 2024-12-06T05:40:04,237 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2266): Finished post open deploy task for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1733463603410.16986c34f3a71270f1d68aff722e562a. 2024-12-06T05:40:04,237 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(153): Opened testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1733463603410.16986c34f3a71270f1d68aff722e562a. 2024-12-06T05:40:04,237 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] handler.AssignRegionHandler(132): Open testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1733463603410.73b074a80469c17ad4213657d37d496f. 2024-12-06T05:40:04,238 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.HRegion(7752): Opening region: {ENCODED => 73b074a80469c17ad4213657d37d496f, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1733463603410.73b074a80469c17ad4213657d37d496f.', STARTKEY => 'i', ENDKEY => 'j'} 2024-12-06T05:40:04,238 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=16986c34f3a71270f1d68aff722e562a, regionState=OPEN, openSeqNum=2, regionLocation=b543b351fe95,35247,1733463601969 2024-12-06T05:40:04,238 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRegionAssignmentAfterMasterRecoveryDueToZKExpiry 73b074a80469c17ad4213657d37d496f 2024-12-06T05:40:04,238 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1733463603410.73b074a80469c17ad4213657d37d496f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:40:04,238 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.HRegion(7794): checking encryption for 73b074a80469c17ad4213657d37d496f 2024-12-06T05:40:04,238 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.HRegion(7797): checking classloading for 73b074a80469c17ad4213657d37d496f 2024-12-06T05:40:04,238 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2266): Finished post open deploy task for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1733463603410.ced409d1a7981960060534bbb519e5d7. 2024-12-06T05:40:04,238 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(153): Opened testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1733463603410.ced409d1a7981960060534bbb519e5d7. 2024-12-06T05:40:04,238 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1733463603410.23e1669d19d0104a438a531c9f1c918d. 2024-12-06T05:40:04,239 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 23e1669d19d0104a438a531c9f1c918d, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1733463603410.23e1669d19d0104a438a531c9f1c918d.', STARTKEY => 'b', ENDKEY => 'c'} 2024-12-06T05:40:04,239 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRegionAssignmentAfterMasterRecoveryDueToZKExpiry 23e1669d19d0104a438a531c9f1c918d 2024-12-06T05:40:04,239 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1733463603410.23e1669d19d0104a438a531c9f1c918d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:40:04,239 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 23e1669d19d0104a438a531c9f1c918d 2024-12-06T05:40:04,239 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 23e1669d19d0104a438a531c9f1c918d 2024-12-06T05:40:04,240 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ced409d1a7981960060534bbb519e5d7, regionState=OPEN, openSeqNum=2, regionLocation=b543b351fe95,38779,1733463601993 2024-12-06T05:40:04,241 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2266): Finished post open deploy task for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1733463603410.a8d0bc8f1ee440609a75868892f81f67. 2024-12-06T05:40:04,241 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(153): Opened testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1733463603410.a8d0bc8f1ee440609a75868892f81f67. 2024-12-06T05:40:04,241 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1733463603410.8b92523d97289af37e0b4eb02850f93c. 2024-12-06T05:40:04,241 INFO [StoreOpener-73b074a80469c17ad4213657d37d496f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 73b074a80469c17ad4213657d37d496f 2024-12-06T05:40:04,241 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 8b92523d97289af37e0b4eb02850f93c, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1733463603410.8b92523d97289af37e0b4eb02850f93c.', STARTKEY => 'g', ENDKEY => 'h'} 2024-12-06T05:40:04,242 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=a8d0bc8f1ee440609a75868892f81f67, regionState=OPEN, openSeqNum=2, regionLocation=b543b351fe95,39035,1733463603336 2024-12-06T05:40:04,242 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRegionAssignmentAfterMasterRecoveryDueToZKExpiry 8b92523d97289af37e0b4eb02850f93c 2024-12-06T05:40:04,242 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1733463603410.8b92523d97289af37e0b4eb02850f93c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:40:04,242 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 8b92523d97289af37e0b4eb02850f93c 2024-12-06T05:40:04,242 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 8b92523d97289af37e0b4eb02850f93c 2024-12-06T05:40:04,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 16986c34f3a71270f1d68aff722e562a, server=b543b351fe95,35247,1733463601969 because future has completed 2024-12-06T05:40:04,243 INFO [StoreOpener-73b074a80469c17ad4213657d37d496f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73b074a80469c17ad4213657d37d496f columnFamilyName info 2024-12-06T05:40:04,243 DEBUG [StoreOpener-73b074a80469c17ad4213657d37d496f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:40:04,244 INFO [StoreOpener-73b074a80469c17ad4213657d37d496f-1 {}] regionserver.HStore(327): Store=73b074a80469c17ad4213657d37d496f/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:40:04,244 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.HRegion(1038): replaying wal for 73b074a80469c17ad4213657d37d496f 2024-12-06T05:40:04,244 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ced409d1a7981960060534bbb519e5d7, server=b543b351fe95,38779,1733463601993 because future has completed 2024-12-06T05:40:04,245 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/73b074a80469c17ad4213657d37d496f 2024-12-06T05:40:04,245 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32775 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=b543b351fe95,39035,1733463603336, table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=a8d0bc8f1ee440609a75868892f81f67. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-06T05:40:04,245 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/73b074a80469c17ad4213657d37d496f 2024-12-06T05:40:04,246 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.HRegion(1048): stopping wal replay for 73b074a80469c17ad4213657d37d496f 2024-12-06T05:40:04,246 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.HRegion(1060): Cleaning up temporary data for 73b074a80469c17ad4213657d37d496f 2024-12-06T05:40:04,246 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure a8d0bc8f1ee440609a75868892f81f67, server=b543b351fe95,39035,1733463603336 because future has completed 2024-12-06T05:40:04,247 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.HRegion(1093): writing seq id for 73b074a80469c17ad4213657d37d496f 2024-12-06T05:40:04,248 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=6 2024-12-06T05:40:04,248 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=5 2024-12-06T05:40:04,248 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 16986c34f3a71270f1d68aff722e562a, server=b543b351fe95,35247,1733463601969 in 173 msec 2024-12-06T05:40:04,249 INFO [StoreOpener-23e1669d19d0104a438a531c9f1c918d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 23e1669d19d0104a438a531c9f1c918d 2024-12-06T05:40:04,250 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=16986c34f3a71270f1d68aff722e562a, ASSIGN in 353 msec 2024-12-06T05:40:04,250 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ced409d1a7981960060534bbb519e5d7, server=b543b351fe95,38779,1733463601993 in 176 msec 2024-12-06T05:40:04,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-06T05:40:04,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure a8d0bc8f1ee440609a75868892f81f67, server=b543b351fe95,39035,1733463603336 in 195 msec 2024-12-06T05:40:04,251 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=ced409d1a7981960060534bbb519e5d7, ASSIGN in 353 msec 2024-12-06T05:40:04,251 INFO [StoreOpener-23e1669d19d0104a438a531c9f1c918d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23e1669d19d0104a438a531c9f1c918d columnFamilyName info 2024-12-06T05:40:04,251 DEBUG [StoreOpener-23e1669d19d0104a438a531c9f1c918d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:40:04,251 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=a8d0bc8f1ee440609a75868892f81f67, ASSIGN in 355 msec 2024-12-06T05:40:04,251 INFO [StoreOpener-23e1669d19d0104a438a531c9f1c918d-1 {}] regionserver.HStore(327): Store=23e1669d19d0104a438a531c9f1c918d/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:40:04,251 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 23e1669d19d0104a438a531c9f1c918d 2024-12-06T05:40:04,252 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/23e1669d19d0104a438a531c9f1c918d 2024-12-06T05:40:04,252 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/23e1669d19d0104a438a531c9f1c918d 2024-12-06T05:40:04,253 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 23e1669d19d0104a438a531c9f1c918d 2024-12-06T05:40:04,253 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 23e1669d19d0104a438a531c9f1c918d 2024-12-06T05:40:04,253 INFO [StoreOpener-8b92523d97289af37e0b4eb02850f93c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 8b92523d97289af37e0b4eb02850f93c 2024-12-06T05:40:04,253 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/73b074a80469c17ad4213657d37d496f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:40:04,254 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 23e1669d19d0104a438a531c9f1c918d 2024-12-06T05:40:04,254 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.HRegion(1114): Opened 73b074a80469c17ad4213657d37d496f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64368339, jitterRate=-0.04083700478076935}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:40:04,254 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 73b074a80469c17ad4213657d37d496f 2024-12-06T05:40:04,254 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.HRegion(1006): Region open journal for 73b074a80469c17ad4213657d37d496f: Running coprocessor pre-open hook at 1733463604238Writing region info on filesystem at 1733463604238Initializing all the Stores at 1733463604240 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463604240Cleaning up temporary data from old regions at 1733463604246 (+6 ms)Running coprocessor post-open hooks at 1733463604254 (+8 ms)Region opened successfully at 1733463604254 2024-12-06T05:40:04,255 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.HRegionServer(2236): Post open deploy tasks for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1733463603410.73b074a80469c17ad4213657d37d496f., pid=25, masterSystemTime=1733463604218 2024-12-06T05:40:04,256 INFO [StoreOpener-8b92523d97289af37e0b4eb02850f93c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b92523d97289af37e0b4eb02850f93c columnFamilyName info 2024-12-06T05:40:04,256 DEBUG [StoreOpener-8b92523d97289af37e0b4eb02850f93c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:40:04,256 INFO [StoreOpener-8b92523d97289af37e0b4eb02850f93c-1 {}] regionserver.HStore(327): Store=8b92523d97289af37e0b4eb02850f93c/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:40:04,256 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 8b92523d97289af37e0b4eb02850f93c 2024-12-06T05:40:04,257 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/8b92523d97289af37e0b4eb02850f93c 2024-12-06T05:40:04,257 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] regionserver.HRegionServer(2266): Finished post open deploy task for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1733463603410.73b074a80469c17ad4213657d37d496f. 2024-12-06T05:40:04,258 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=25}] handler.AssignRegionHandler(153): Opened testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1733463603410.73b074a80469c17ad4213657d37d496f. 2024-12-06T05:40:04,258 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1733463603410.6b7648f7bc7a5892f02675835e253ce4. 2024-12-06T05:40:04,258 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 6b7648f7bc7a5892f02675835e253ce4, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1733463603410.6b7648f7bc7a5892f02675835e253ce4.', STARTKEY => 'd', ENDKEY => 'e'} 2024-12-06T05:40:04,258 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/8b92523d97289af37e0b4eb02850f93c 2024-12-06T05:40:04,258 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/23e1669d19d0104a438a531c9f1c918d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:40:04,258 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 8b92523d97289af37e0b4eb02850f93c 2024-12-06T05:40:04,258 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=73b074a80469c17ad4213657d37d496f, regionState=OPEN, openSeqNum=2, regionLocation=b543b351fe95,35247,1733463601969 2024-12-06T05:40:04,258 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 8b92523d97289af37e0b4eb02850f93c 2024-12-06T05:40:04,258 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRegionAssignmentAfterMasterRecoveryDueToZKExpiry 6b7648f7bc7a5892f02675835e253ce4 2024-12-06T05:40:04,258 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1733463603410.6b7648f7bc7a5892f02675835e253ce4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:40:04,259 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 6b7648f7bc7a5892f02675835e253ce4 2024-12-06T05:40:04,259 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 6b7648f7bc7a5892f02675835e253ce4 2024-12-06T05:40:04,260 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 23e1669d19d0104a438a531c9f1c918d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67633006, jitterRate=0.007810324430465698}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:40:04,261 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 23e1669d19d0104a438a531c9f1c918d 2024-12-06T05:40:04,261 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 23e1669d19d0104a438a531c9f1c918d: Running coprocessor pre-open hook at 1733463604239Writing region info on filesystem at 1733463604239Initializing all the Stores at 1733463604241 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463604241Cleaning up temporary data from old regions at 1733463604253 (+12 ms)Running coprocessor post-open hooks at 1733463604261 (+8 ms)Region opened successfully at 1733463604261 2024-12-06T05:40:04,261 INFO [StoreOpener-6b7648f7bc7a5892f02675835e253ce4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 6b7648f7bc7a5892f02675835e253ce4 2024-12-06T05:40:04,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 73b074a80469c17ad4213657d37d496f, server=b543b351fe95,35247,1733463601969 because future has completed 2024-12-06T05:40:04,261 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1733463603410.23e1669d19d0104a438a531c9f1c918d., pid=24, masterSystemTime=1733463604213 2024-12-06T05:40:04,262 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 8b92523d97289af37e0b4eb02850f93c 2024-12-06T05:40:04,263 INFO [StoreOpener-6b7648f7bc7a5892f02675835e253ce4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b7648f7bc7a5892f02675835e253ce4 columnFamilyName info 2024-12-06T05:40:04,263 DEBUG [StoreOpener-6b7648f7bc7a5892f02675835e253ce4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:40:04,264 INFO [StoreOpener-6b7648f7bc7a5892f02675835e253ce4-1 {}] regionserver.HStore(327): Store=6b7648f7bc7a5892f02675835e253ce4/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:40:04,264 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 6b7648f7bc7a5892f02675835e253ce4 2024-12-06T05:40:04,265 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/8b92523d97289af37e0b4eb02850f93c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:40:04,265 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/6b7648f7bc7a5892f02675835e253ce4 2024-12-06T05:40:04,265 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1733463603410.23e1669d19d0104a438a531c9f1c918d. 2024-12-06T05:40:04,265 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1733463603410.23e1669d19d0104a438a531c9f1c918d. 2024-12-06T05:40:04,266 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 8b92523d97289af37e0b4eb02850f93c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74928341, jitterRate=0.11651928722858429}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:40:04,266 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/6b7648f7bc7a5892f02675835e253ce4 2024-12-06T05:40:04,266 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8b92523d97289af37e0b4eb02850f93c 2024-12-06T05:40:04,266 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 8b92523d97289af37e0b4eb02850f93c: Running coprocessor pre-open hook at 1733463604242Writing region info on filesystem at 1733463604242Initializing all the Stores at 1733463604245 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463604245Cleaning up temporary data from old regions at 1733463604258 (+13 ms)Running coprocessor post-open hooks at 1733463604266 (+8 ms)Region opened successfully at 1733463604266 2024-12-06T05:40:04,266 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=23e1669d19d0104a438a531c9f1c918d, regionState=OPEN, openSeqNum=2, regionLocation=b543b351fe95,38779,1733463601993 2024-12-06T05:40:04,266 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] handler.AssignRegionHandler(132): Open testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1733463603410.5bfbbde54676defb425c75f2d409c104. 2024-12-06T05:40:04,267 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(7752): Opening region: {ENCODED => 5bfbbde54676defb425c75f2d409c104, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1733463603410.5bfbbde54676defb425c75f2d409c104.', STARTKEY => 'e', ENDKEY => 'f'} 2024-12-06T05:40:04,267 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRegionAssignmentAfterMasterRecoveryDueToZKExpiry 5bfbbde54676defb425c75f2d409c104 2024-12-06T05:40:04,267 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1733463603410.5bfbbde54676defb425c75f2d409c104.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:40:04,267 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(7794): checking encryption for 5bfbbde54676defb425c75f2d409c104 2024-12-06T05:40:04,267 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(7797): checking classloading for 5bfbbde54676defb425c75f2d409c104 2024-12-06T05:40:04,267 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=14 2024-12-06T05:40:04,268 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1733463603410.8b92523d97289af37e0b4eb02850f93c., pid=17, masterSystemTime=1733463604206 2024-12-06T05:40:04,268 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 73b074a80469c17ad4213657d37d496f, server=b543b351fe95,35247,1733463601969 in 192 msec 2024-12-06T05:40:04,268 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 6b7648f7bc7a5892f02675835e253ce4 2024-12-06T05:40:04,268 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 6b7648f7bc7a5892f02675835e253ce4 2024-12-06T05:40:04,270 INFO [StoreOpener-5bfbbde54676defb425c75f2d409c104-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 5bfbbde54676defb425c75f2d409c104 2024-12-06T05:40:04,270 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23e1669d19d0104a438a531c9f1c918d, server=b543b351fe95,38779,1733463601993 because future has completed 2024-12-06T05:40:04,270 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 6b7648f7bc7a5892f02675835e253ce4 2024-12-06T05:40:04,270 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1733463603410.8b92523d97289af37e0b4eb02850f93c. 2024-12-06T05:40:04,271 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1733463603410.8b92523d97289af37e0b4eb02850f93c. 2024-12-06T05:40:04,271 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(132): Open testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1733463603410.b20ebfffac3488c42f88544a5529f43f. 2024-12-06T05:40:04,271 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7752): Opening region: {ENCODED => b20ebfffac3488c42f88544a5529f43f, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1733463603410.b20ebfffac3488c42f88544a5529f43f.', STARTKEY => 'f', ENDKEY => 'g'} 2024-12-06T05:40:04,271 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=73b074a80469c17ad4213657d37d496f, ASSIGN in 373 msec 2024-12-06T05:40:04,271 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRegionAssignmentAfterMasterRecoveryDueToZKExpiry b20ebfffac3488c42f88544a5529f43f 2024-12-06T05:40:04,271 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=8b92523d97289af37e0b4eb02850f93c, regionState=OPEN, openSeqNum=2, regionLocation=b543b351fe95,39035,1733463603336 2024-12-06T05:40:04,271 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1733463603410.b20ebfffac3488c42f88544a5529f43f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:40:04,271 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7794): checking encryption for b20ebfffac3488c42f88544a5529f43f 2024-12-06T05:40:04,271 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7797): checking classloading for b20ebfffac3488c42f88544a5529f43f 2024-12-06T05:40:04,273 INFO [StoreOpener-b20ebfffac3488c42f88544a5529f43f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region b20ebfffac3488c42f88544a5529f43f 2024-12-06T05:40:04,274 INFO [StoreOpener-5bfbbde54676defb425c75f2d409c104-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bfbbde54676defb425c75f2d409c104 columnFamilyName info 2024-12-06T05:40:04,274 DEBUG [StoreOpener-5bfbbde54676defb425c75f2d409c104-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:40:04,275 INFO [StoreOpener-b20ebfffac3488c42f88544a5529f43f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b20ebfffac3488c42f88544a5529f43f columnFamilyName info 2024-12-06T05:40:04,275 DEBUG [StoreOpener-b20ebfffac3488c42f88544a5529f43f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:40:04,275 INFO [StoreOpener-5bfbbde54676defb425c75f2d409c104-1 {}] regionserver.HStore(327): Store=5bfbbde54676defb425c75f2d409c104/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:40:04,275 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b92523d97289af37e0b4eb02850f93c, server=b543b351fe95,39035,1733463603336 because future has completed 2024-12-06T05:40:04,275 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1038): replaying wal for 5bfbbde54676defb425c75f2d409c104 2024-12-06T05:40:04,276 INFO [StoreOpener-b20ebfffac3488c42f88544a5529f43f-1 {}] regionserver.HStore(327): Store=b20ebfffac3488c42f88544a5529f43f/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:40:04,276 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1038): replaying wal for b20ebfffac3488c42f88544a5529f43f 2024-12-06T05:40:04,276 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/5bfbbde54676defb425c75f2d409c104 2024-12-06T05:40:04,276 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/b20ebfffac3488c42f88544a5529f43f 2024-12-06T05:40:04,276 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=7 2024-12-06T05:40:04,277 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/5bfbbde54676defb425c75f2d409c104 2024-12-06T05:40:04,277 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/b20ebfffac3488c42f88544a5529f43f 2024-12-06T05:40:04,277 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1048): stopping wal replay for b20ebfffac3488c42f88544a5529f43f 2024-12-06T05:40:04,277 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1048): stopping wal replay for 5bfbbde54676defb425c75f2d409c104 2024-12-06T05:40:04,277 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1060): Cleaning up temporary data for b20ebfffac3488c42f88544a5529f43f 2024-12-06T05:40:04,277 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1060): Cleaning up temporary data for 5bfbbde54676defb425c75f2d409c104 2024-12-06T05:40:04,278 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/6b7648f7bc7a5892f02675835e253ce4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:40:04,278 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 6b7648f7bc7a5892f02675835e253ce4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67432731, jitterRate=0.004825994372367859}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:40:04,278 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6b7648f7bc7a5892f02675835e253ce4 2024-12-06T05:40:04,278 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 6b7648f7bc7a5892f02675835e253ce4: Running coprocessor pre-open hook at 1733463604259Writing region info on filesystem at 1733463604259Initializing all the Stores at 1733463604261 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463604261Cleaning up temporary data from old regions at 1733463604268 (+7 ms)Running coprocessor post-open hooks at 1733463604278 (+10 ms)Region opened successfully at 1733463604278 2024-12-06T05:40:04,279 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1093): writing seq id for b20ebfffac3488c42f88544a5529f43f 2024-12-06T05:40:04,279 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1093): writing seq id for 5bfbbde54676defb425c75f2d409c104 2024-12-06T05:40:04,281 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/b20ebfffac3488c42f88544a5529f43f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:40:04,281 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/5bfbbde54676defb425c75f2d409c104/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:40:04,282 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1114): Opened b20ebfffac3488c42f88544a5529f43f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75280602, jitterRate=0.12176838517189026}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:40:04,282 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1114): Opened 5bfbbde54676defb425c75f2d409c104; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74447531, jitterRate=0.1093546599149704}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:40:04,282 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b20ebfffac3488c42f88544a5529f43f 2024-12-06T05:40:04,282 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5bfbbde54676defb425c75f2d409c104 2024-12-06T05:40:04,282 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1006): Region open journal for 5bfbbde54676defb425c75f2d409c104: Running coprocessor pre-open hook at 1733463604267Writing region info on filesystem at 1733463604267Initializing all the Stores at 1733463604268 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463604268Cleaning up temporary data from old regions at 1733463604277 (+9 ms)Running coprocessor post-open hooks at 1733463604282 (+5 ms)Region opened successfully at 1733463604282 2024-12-06T05:40:04,282 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1006): Region open journal for b20ebfffac3488c42f88544a5529f43f: Running coprocessor pre-open hook at 1733463604272Writing region info on filesystem at 1733463604272Initializing all the Stores at 1733463604272Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463604272Cleaning up temporary data from old regions at 1733463604277 (+5 ms)Running coprocessor post-open hooks at 1733463604282 (+5 ms)Region opened successfully at 1733463604282 2024-12-06T05:40:04,283 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2236): Post open deploy tasks for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1733463603410.b20ebfffac3488c42f88544a5529f43f., pid=19, masterSystemTime=1733463604206 2024-12-06T05:40:04,285 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1733463603410.6b7648f7bc7a5892f02675835e253ce4., pid=21, masterSystemTime=1733463604218 2024-12-06T05:40:04,287 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegionServer(2236): Post open deploy tasks for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1733463603410.5bfbbde54676defb425c75f2d409c104., pid=22, masterSystemTime=1733463604213 2024-12-06T05:40:04,288 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2266): Finished post open deploy task for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1733463603410.b20ebfffac3488c42f88544a5529f43f. 2024-12-06T05:40:04,288 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(153): Opened testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1733463603410.b20ebfffac3488c42f88544a5529f43f. 2024-12-06T05:40:04,288 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(132): Open testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1733463603410.c0251d107f2e4c28a8d176e8af25b695. 2024-12-06T05:40:04,288 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7752): Opening region: {ENCODED => c0251d107f2e4c28a8d176e8af25b695, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1733463603410.c0251d107f2e4c28a8d176e8af25b695.', STARTKEY => 'c', ENDKEY => 'd'} 2024-12-06T05:40:04,288 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRegionAssignmentAfterMasterRecoveryDueToZKExpiry c0251d107f2e4c28a8d176e8af25b695 2024-12-06T05:40:04,289 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1733463603410.c0251d107f2e4c28a8d176e8af25b695.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:40:04,289 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7794): checking encryption for c0251d107f2e4c28a8d176e8af25b695 2024-12-06T05:40:04,289 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7797): checking classloading for c0251d107f2e4c28a8d176e8af25b695 2024-12-06T05:40:04,290 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 23e1669d19d0104a438a531c9f1c918d, server=b543b351fe95,38779,1733463601993 in 201 msec 2024-12-06T05:40:04,290 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1733463603410.6b7648f7bc7a5892f02675835e253ce4. 2024-12-06T05:40:04,290 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1733463603410.6b7648f7bc7a5892f02675835e253ce4. 2024-12-06T05:40:04,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=23e1669d19d0104a438a531c9f1c918d, ASSIGN in 392 msec 2024-12-06T05:40:04,292 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=12 2024-12-06T05:40:04,292 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 8b92523d97289af37e0b4eb02850f93c, server=b543b351fe95,39035,1733463603336 in 222 msec 2024-12-06T05:40:04,292 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=b20ebfffac3488c42f88544a5529f43f, regionState=OPEN, openSeqNum=2, regionLocation=b543b351fe95,39035,1733463603336 2024-12-06T05:40:04,292 INFO [StoreOpener-c0251d107f2e4c28a8d176e8af25b695-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region c0251d107f2e4c28a8d176e8af25b695 2024-12-06T05:40:04,293 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=6b7648f7bc7a5892f02675835e253ce4, regionState=OPEN, openSeqNum=2, regionLocation=b543b351fe95,35247,1733463601969 2024-12-06T05:40:04,294 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegionServer(2266): Finished post open deploy task for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1733463603410.5bfbbde54676defb425c75f2d409c104. 2024-12-06T05:40:04,294 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] handler.AssignRegionHandler(153): Opened testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1733463603410.5bfbbde54676defb425c75f2d409c104. 2024-12-06T05:40:04,294 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1733463603410.cc3edcebfba16936afa2293e025fd871. 2024-12-06T05:40:04,295 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => cc3edcebfba16936afa2293e025fd871, NAME => 'testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1733463603410.cc3edcebfba16936afa2293e025fd871.', STARTKEY => 'j', ENDKEY => ''} 2024-12-06T05:40:04,295 INFO [StoreOpener-c0251d107f2e4c28a8d176e8af25b695-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0251d107f2e4c28a8d176e8af25b695 columnFamilyName info 2024-12-06T05:40:04,295 DEBUG [StoreOpener-c0251d107f2e4c28a8d176e8af25b695-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:40:04,295 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRegionAssignmentAfterMasterRecoveryDueToZKExpiry cc3edcebfba16936afa2293e025fd871 2024-12-06T05:40:04,295 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1733463603410.cc3edcebfba16936afa2293e025fd871.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:40:04,295 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for cc3edcebfba16936afa2293e025fd871 2024-12-06T05:40:04,295 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for cc3edcebfba16936afa2293e025fd871 2024-12-06T05:40:04,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure b20ebfffac3488c42f88544a5529f43f, server=b543b351fe95,39035,1733463603336 because future has completed 2024-12-06T05:40:04,297 INFO [StoreOpener-c0251d107f2e4c28a8d176e8af25b695-1 {}] regionserver.HStore(327): Store=c0251d107f2e4c28a8d176e8af25b695/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:40:04,297 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1038): replaying wal for c0251d107f2e4c28a8d176e8af25b695 2024-12-06T05:40:04,299 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/c0251d107f2e4c28a8d176e8af25b695 2024-12-06T05:40:04,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6b7648f7bc7a5892f02675835e253ce4, server=b543b351fe95,35247,1733463601969 because future has completed 2024-12-06T05:40:04,299 INFO [StoreOpener-cc3edcebfba16936afa2293e025fd871-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region cc3edcebfba16936afa2293e025fd871 2024-12-06T05:40:04,299 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/c0251d107f2e4c28a8d176e8af25b695 2024-12-06T05:40:04,299 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1048): stopping wal replay for c0251d107f2e4c28a8d176e8af25b695 2024-12-06T05:40:04,300 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1060): Cleaning up temporary data for c0251d107f2e4c28a8d176e8af25b695 2024-12-06T05:40:04,300 INFO [StoreOpener-cc3edcebfba16936afa2293e025fd871-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc3edcebfba16936afa2293e025fd871 columnFamilyName info 2024-12-06T05:40:04,300 DEBUG [StoreOpener-cc3edcebfba16936afa2293e025fd871-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:40:04,301 INFO [StoreOpener-cc3edcebfba16936afa2293e025fd871-1 {}] regionserver.HStore(327): Store=cc3edcebfba16936afa2293e025fd871/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:40:04,301 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for cc3edcebfba16936afa2293e025fd871 2024-12-06T05:40:04,301 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1093): writing seq id for c0251d107f2e4c28a8d176e8af25b695 2024-12-06T05:40:04,302 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=8b92523d97289af37e0b4eb02850f93c, ASSIGN in 397 msec 2024-12-06T05:40:04,302 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=5bfbbde54676defb425c75f2d409c104, regionState=OPEN, openSeqNum=2, regionLocation=b543b351fe95,38779,1733463601993 2024-12-06T05:40:04,302 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/cc3edcebfba16936afa2293e025fd871 2024-12-06T05:40:04,302 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/cc3edcebfba16936afa2293e025fd871 2024-12-06T05:40:04,303 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for cc3edcebfba16936afa2293e025fd871 2024-12-06T05:40:04,303 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for cc3edcebfba16936afa2293e025fd871 2024-12-06T05:40:04,306 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/c0251d107f2e4c28a8d176e8af25b695/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:40:04,306 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for cc3edcebfba16936afa2293e025fd871 2024-12-06T05:40:04,306 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1114): Opened c0251d107f2e4c28a8d176e8af25b695; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64475824, jitterRate=-0.03923535346984863}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:40:04,306 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c0251d107f2e4c28a8d176e8af25b695 2024-12-06T05:40:04,307 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1006): Region open journal for c0251d107f2e4c28a8d176e8af25b695: Running coprocessor pre-open hook at 1733463604289Writing region info on filesystem at 1733463604289Initializing all the Stores at 1733463604291 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463604292 (+1 ms)Cleaning up temporary data from old regions at 1733463604300 (+8 ms)Running coprocessor post-open hooks at 1733463604306 (+6 ms)Region opened successfully at 1733463604307 (+1 ms) 2024-12-06T05:40:04,308 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2236): Post open deploy tasks for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1733463603410.c0251d107f2e4c28a8d176e8af25b695., pid=20, masterSystemTime=1733463604206 2024-12-06T05:40:04,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bfbbde54676defb425c75f2d409c104, server=b543b351fe95,38779,1733463601993 because future has completed 2024-12-06T05:40:04,310 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/data/default/testRegionAssignmentAfterMasterRecoveryDueToZKExpiry/cc3edcebfba16936afa2293e025fd871/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:40:04,310 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened cc3edcebfba16936afa2293e025fd871; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67463547, jitterRate=0.005285188555717468}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:40:04,310 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cc3edcebfba16936afa2293e025fd871 2024-12-06T05:40:04,311 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for cc3edcebfba16936afa2293e025fd871: Running coprocessor pre-open hook at 1733463604295Writing region info on filesystem at 1733463604295Initializing all the Stores at 1733463604297 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733463604297Cleaning up temporary data from old regions at 1733463604303 (+6 ms)Running coprocessor post-open hooks at 1733463604310 (+7 ms)Region opened successfully at 1733463604310 2024-12-06T05:40:04,311 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=11 2024-12-06T05:40:04,311 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure b20ebfffac3488c42f88544a5529f43f, server=b543b351fe95,39035,1733463603336 in 245 msec 2024-12-06T05:40:04,312 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1733463603410.cc3edcebfba16936afa2293e025fd871., pid=18, masterSystemTime=1733463604213 2024-12-06T05:40:04,312 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2266): Finished post open deploy task for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1733463603410.c0251d107f2e4c28a8d176e8af25b695. 2024-12-06T05:40:04,312 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(153): Opened testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1733463603410.c0251d107f2e4c28a8d176e8af25b695. 2024-12-06T05:40:04,313 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=9 2024-12-06T05:40:04,314 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 6b7648f7bc7a5892f02675835e253ce4, server=b543b351fe95,35247,1733463601969 in 238 msec 2024-12-06T05:40:04,314 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=b20ebfffac3488c42f88544a5529f43f, ASSIGN in 416 msec 2024-12-06T05:40:04,315 DEBUG [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1733463603410.cc3edcebfba16936afa2293e025fd871. 2024-12-06T05:40:04,315 INFO [RS_OPEN_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1733463603410.cc3edcebfba16936afa2293e025fd871. 2024-12-06T05:40:04,316 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=cc3edcebfba16936afa2293e025fd871, regionState=OPEN, openSeqNum=2, regionLocation=b543b351fe95,38779,1733463601993 2024-12-06T05:40:04,316 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=c0251d107f2e4c28a8d176e8af25b695, regionState=OPEN, openSeqNum=2, regionLocation=b543b351fe95,39035,1733463603336 2024-12-06T05:40:04,319 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=10 2024-12-06T05:40:04,319 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 5bfbbde54676defb425c75f2d409c104, server=b543b351fe95,38779,1733463601993 in 244 msec 2024-12-06T05:40:04,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0251d107f2e4c28a8d176e8af25b695, server=b543b351fe95,39035,1733463603336 because future has completed 2024-12-06T05:40:04,321 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=6b7648f7bc7a5892f02675835e253ce4, ASSIGN in 420 msec 2024-12-06T05:40:04,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=5bfbbde54676defb425c75f2d409c104, ASSIGN in 424 msec 2024-12-06T05:40:04,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-23-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure cc3edcebfba16936afa2293e025fd871, server=b543b351fe95,38779,1733463601993 because future has completed 2024-12-06T05:40:04,326 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=8 2024-12-06T05:40:04,327 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure c0251d107f2e4c28a8d176e8af25b695, server=b543b351fe95,39035,1733463603336 in 262 msec 2024-12-06T05:40:04,328 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=15 2024-12-06T05:40:04,328 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure cc3edcebfba16936afa2293e025fd871, server=b543b351fe95,38779,1733463601993 in 267 msec 2024-12-06T05:40:04,328 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=c0251d107f2e4c28a8d176e8af25b695, ASSIGN in 431 msec 2024-12-06T05:40:04,331 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=4 2024-12-06T05:40:04,331 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, region=cc3edcebfba16936afa2293e025fd871, ASSIGN in 433 msec 2024-12-06T05:40:04,332 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T05:40:04,332 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRegionAssignmentAfterMasterRecoveryDueToZKExpiry","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733463604332"}]},"ts":"1733463604332"} 2024-12-06T05:40:04,337 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry, state=ENABLED in hbase:meta 2024-12-06T05:40:04,338 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T05:40:04,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry in 928 msec 2024-12-06T05:40:04,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T05:40:04,549 INFO [RPCClient-NioEventLoopGroup-6-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRegionAssignmentAfterMasterRecoveryDueToZKExpiry completed 2024-12-06T05:40:04,550 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T05:40:04,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:40:04,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:04,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x100ae0a5c73001b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:40:04,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x100ae0a5c73001b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:04,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:40:04,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:04,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x100ae0a5c73001c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:40:04,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x100ae0a5c73001c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:04,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39035-0x100ae0a5c73001e, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:40:04,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39035-0x100ae0a5c73001e, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:04,552 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:40:04,552 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35247-0x100ae0a5c73001b, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:40:04,552 DEBUG [zk-event-processor-pool-0 {}] master.ActiveMasterManager(204): No master available. Notifying waiting threads 2024-12-06T05:40:04,552 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38779-0x100ae0a5c73001c, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:40:04,552 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39035-0x100ae0a5c73001e, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:40:04,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x100ae0a5c73001c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T05:40:04,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39035-0x100ae0a5c73001e, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T05:40:04,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T05:40:04,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:04,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x100ae0a5c73001b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T05:40:04,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x100ae0a5c73001b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:04,554 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T05:40:04,554 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T05:40:04,555 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b543b351fe95,46477,1733463601919 from backup master directory 2024-12-06T05:40:04,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39035-0x100ae0a5c73001e, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:40:04,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x100ae0a5c73001c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:40:04,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x100ae0a5c73001b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:40:04,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b543b351fe95,46477,1733463601919 2024-12-06T05:40:04,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:40:04,556 WARN [master/b543b351fe95:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T05:40:04,556 INFO [master/b543b351fe95:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b543b351fe95,46477,1733463601919 2024-12-06T05:40:04,567 INFO [master/b543b351fe95:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:40:04,567 INFO [master/b543b351fe95:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-06T05:40:04,577 INFO [master/b543b351fe95:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=2) cost 10ms. 2024-12-06T05:40:04,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741931_1108 (size=196) 2024-12-06T05:40:04,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741931_1108 (size=196) 2024-12-06T05:40:04,588 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T05:40:04,588 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-06T05:40:04,589 INFO [master/b543b351fe95:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:40:04,592 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionAssignmentAfterMasterRecoveryDueToZKExpiry 2024-12-06T05:40:04,592 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionAssignmentAfterMasterRecoveryDueToZKExpiry Metrics about Tables on a single HBase RegionServer 2024-12-06T05:40:04,593 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testLogSplittingAfterMasterRecoveryDueToZKExpiry 2024-12-06T05:40:04,593 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testLogSplittingAfterMasterRecoveryDueToZKExpiry Metrics about Tables on a single HBase RegionServer 2024-12-06T05:40:04,598 DEBUG [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(342): old store file tracker DEFAULT is the same with new store file tracker, skip migration 2024-12-06T05:40:04,601 INFO [master/b543b351fe95:0:becomeActiveMaster {}] region.MasterRegion(316): Renamed hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,32775,1733463601942 to hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,32775,1733463601942-dead as it is dead 2024-12-06T05:40:04,601 INFO [master/b543b351fe95:0:becomeActiveMaster {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,32775,1733463601942-dead/b543b351fe95%2C32775%2C1733463601942.1733463602480 2024-12-06T05:40:04,602 WARN [IPC Server handler 0 on default port 34453 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,32775,1733463601942-dead/b543b351fe95%2C32775%2C1733463601942.1733463602480 has not been closed. Lease recovery is in progress. RecoveryId = 1109 for block blk_1073741912_1089 2024-12-06T05:40:04,602 INFO [master/b543b351fe95:0:becomeActiveMaster {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,32775,1733463601942-dead/b543b351fe95%2C32775%2C1733463601942.1733463602480 after 1ms 2024-12-06T05:40:04,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:40:04,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:40:04,653 ERROR [Time-limited test {}] master.HMaster(3218): Master server abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-06T05:40:04,653 ERROR [Time-limited test {}] master.HMaster(3223): ***** ABORTING master b543b351fe95,32775,1733463601942: Test recovery from zk session expired ***** org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired at org.apache.hadoop.hbase.TestZooKeeper.testRegionAssignmentAfterMasterRecoveryDueToZKExpiry(TestZooKeeper.java:196) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:40:04,654 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master 'b543b351fe95,32775,1733463601942' ***** 2024-12-06T05:40:04,654 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: Stopped by Time-limited test 2024-12-06T05:40:04,654 INFO [M:1;b543b351fe95:32775 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T05:40:04,654 INFO [M:1;b543b351fe95:32775 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:1;b543b351fe95:32775. 2024-12-06T05:40:04,654 DEBUG [M:1;b543b351fe95:32775 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T05:40:04,654 DEBUG [M:1;b543b351fe95:32775 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:40:04,654 INFO [M:1;b543b351fe95:32775 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T05:40:04,654 DEBUG [M:1;b543b351fe95:32775 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-06T05:40:04,654 DEBUG [M:1;b543b351fe95:32775 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-06T05:40:04,654 WARN [Time-limited test {}] zookeeper.ZKUtil(351): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/master org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /hbase/master at org.apache.zookeeper.KeeperException.create(KeeperException.java:134) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1972) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:368) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:356) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.getData(ZKUtil.java:343) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.MasterAddressTracker.getMasterAddress(MasterAddressTracker.java:171) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.ActiveMasterManager.stop(ActiveMasterManager.java:342) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.stop(HMaster.java:3327) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.stopMaster(HMaster.java:3315) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.abort(HMaster.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.TestZooKeeper.testRegionAssignmentAfterMasterRecoveryDueToZKExpiry(TestZooKeeper.java:196) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:40:04,654 INFO [M:1;b543b351fe95:32775 {}] hbase.ChoreService(370): Chore service for: master/b543b351fe95:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-06T05:40:04,655 INFO [M:1;b543b351fe95:32775 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T05:40:04,655 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-06T05:40:04,655 DEBUG [M:1;b543b351fe95:32775 {}] master.HMaster(1795): Stopping service threads 2024-12-06T05:40:04,655 INFO [M:1;b543b351fe95:32775 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-06T05:40:04,655 INFO [M:1;b543b351fe95:32775 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-06T05:40:04,655 ERROR [Time-limited test {}] zookeeper.ZKWatcher(711): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Received unexpected KeeperException, re-throwing exception org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /hbase/master at org.apache.zookeeper.KeeperException.create(KeeperException.java:134) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1972) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:368) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:356) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.getData(ZKUtil.java:343) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.MasterAddressTracker.getMasterAddress(MasterAddressTracker.java:171) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.ActiveMasterManager.stop(ActiveMasterManager.java:342) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.stop(HMaster.java:3327) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.stopMaster(HMaster.java:3315) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.abort(HMaster.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.TestZooKeeper.testRegionAssignmentAfterMasterRecoveryDueToZKExpiry(TestZooKeeper.java:196) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:40:04,655 DEBUG [master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.small.0-1733463602553 {}] cleaner.HFileCleaner(306): Exit Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.small.0-1733463602553,5,FailOnTimeoutGroup] 2024-12-06T05:40:04,655 DEBUG [Time-limited test {}] master.ActiveMasterManager(353): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = Session expired for /hbase/master 2024-12-06T05:40:04,656 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T05:40:04,656 ERROR [Time-limited test {}] master.HMaster(3225): ***** ABORTING master b543b351fe95,46477,1733463601919: killAll ***** 2024-12-06T05:40:04,656 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master 'b543b351fe95,46477,1733463601919' ***** 2024-12-06T05:40:04,656 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: Stopped by Time-limited test 2024-12-06T05:40:04,656 DEBUG [master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.large.0-1733463602551 {}] cleaner.HFileCleaner(306): Exit Thread[master/b543b351fe95:0:becomeActiveMaster-HFileCleaner.large.0-1733463602551,5,FailOnTimeoutGroup] 2024-12-06T05:40:04,656 INFO [M:0;b543b351fe95:46477 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T05:40:04,656 INFO [M:0;b543b351fe95:46477 {}] hbase.ChoreService(370): Chore service for: master/b543b351fe95:0 had [] on shutdown 2024-12-06T05:40:04,656 INFO [M:0;b543b351fe95:46477 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T05:40:04,656 DEBUG [M:0;b543b351fe95:46477 {}] master.HMaster(1795): Stopping service threads 2024-12-06T05:40:04,656 INFO [M:1;b543b351fe95:32775 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=true 2024-12-06T05:40:04,657 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-06T05:40:04,657 WARN [M:1;b543b351fe95:32775 {}] zookeeper.ZKUtil(351): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Unable to get data of znode /hbase/master org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /hbase/master at org.apache.zookeeper.KeeperException.create(KeeperException.java:134) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1972) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:368) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:356) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.getData(ZKUtil.java:343) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.MasterAddressTracker.getMasterAddress(MasterAddressTracker.java:171) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.ActiveMasterManager.stop(ActiveMasterManager.java:342) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.stopServiceThreads(HMaster.java:1809) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:631) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:40:04,657 ERROR [M:1;b543b351fe95:32775 {}] zookeeper.ZKWatcher(711): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Received unexpected KeeperException, re-throwing exception org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /hbase/master at org.apache.zookeeper.KeeperException.create(KeeperException.java:134) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1972) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:368) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:356) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.getData(ZKUtil.java:343) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.MasterAddressTracker.getMasterAddress(MasterAddressTracker.java:171) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.ActiveMasterManager.stop(ActiveMasterManager.java:342) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.stopServiceThreads(HMaster.java:1809) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:631) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:40:04,657 DEBUG [M:1;b543b351fe95:32775 {}] master.ActiveMasterManager(353): master:32775-0x100ae0a5c73001a, quorum=127.0.0.1:58086, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = Session expired for /hbase/master 2024-12-06T05:40:04,657 INFO [M:1;b543b351fe95:32775 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/.lastflushedseqids 2024-12-06T05:40:04,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:40:04,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x100ae0a5c73001c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:40:04,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39035-0x100ae0a5c73001e, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:40:04,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:04,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39035-0x100ae0a5c73001e, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:04,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x100ae0a5c73001c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:04,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x100ae0a5c73001b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:40:04,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x100ae0a5c73001b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:04,658 INFO [M:0;b543b351fe95:46477 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46477 2024-12-06T05:40:04,658 INFO [M:0;b543b351fe95:46477 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T05:40:04,658 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39035-0x100ae0a5c73001e, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:40:04,658 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38779-0x100ae0a5c73001c, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:40:04,659 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35247-0x100ae0a5c73001b, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:40:04,659 DEBUG [Time-limited test {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-12-06T05:40:04,659 DEBUG [Time-limited test {}] master.ActiveMasterManager(353): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-12-06T05:40:04,659 DEBUG [Time-limited test {}] master.HMaster(3213): Abort called but aborted=true, stopped=true 2024-12-06T05:40:04,659 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server b543b351fe95,35247,1733463601969: killAll ***** 2024-12-06T05:40:04,659 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-06T05:40:04,659 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-06T05:40:04,659 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-06T05:40:04,660 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-06T05:40:04,660 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-06T05:40:04,662 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 617071008 }, "NonHeapMemoryUsage": { "committed": 171114496, "init": 7667712, "max": -1, "used": 168572640 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 36454, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 1, "ProcessCallTime_num_ops": 120, "ProcessCallTime_min": 0, "ProcessCallTime_max": 15, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 1, "ProcessCallTime_75th_percentile": 3, "ProcessCallTime_90th_percentile": 6, "ProcessCallTime_95th_percentile": 6, "ProcessCallTime_98th_percentile": 13, "ProcessCallTime_99th_percentile": 15, "ProcessCallTime_99.9th_percentile": 15, "ProcessCallTime_TimeRangeCount_0-1": 32, "ProcessCallTime_TimeRangeCount_1-3": 18, "ProcessCallTime_TimeRangeCount_3-10": 10, "ProcessCallTime_TimeRangeCount_10-30": 3, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 120, "QueueCallTime_min": 0, "QueueCallTime_max": 19, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 1, "QueueCallTime_95th_percentile": 5, "QueueCallTime_98th_percentile": 15, "QueueCallTime_99th_percentile": 19, "QueueCallTime_99.9th_percentile": 19, "QueueCallTime_TimeRangeCount_0-1": 58, "QueueCallTime_TimeRangeCount_1-3": 1, "QueueCallTime_TimeRangeCount_3-10": 1, "QueueCallTime_TimeRangeCount_10-30": 3, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 120, "TotalCallTime_min": 0, "TotalCallTime_max": 32, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 3, "TotalCallTime_90th_percentile": 6, "TotalCallTime_95th_percentile": 15, "TotalCallTime_98th_percentile": 19, "TotalCallTime_99th_percentile": 32, "TotalCallTime_99.9th_percentile": 32, "TotalCallTime_TimeRangeCount_0-1": 28, "TotalCallTime_TimeRangeCount_1-3": 21, "TotalCallTime_TimeRangeCount_3-10": 8, "TotalCallTime_TimeRangeCount_10-30": 5, "TotalCallTime_TimeRangeCount_30-100": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 120, "ResponseSize_min": 0, "ResponseSize_max": 6999, "ResponseSize_mean": 837, "ResponseSize_25th_percentile": 1, "ResponseSize_median": 3, "ResponseSize_75th_percentile": 177, "ResponseSize_90th_percentile": 4326, "ResponseSize_95th_percentile": 5662, "ResponseSize_98th_percentile": 6464, "ResponseSize_99th_percentile": 6731, "ResponseSize_99.9th_percentile": 6972, "ResponseSize_SizeRangeCount_0-10": 36, "ResponseSize_SizeRangeCount_10-100": 9, "ResponseSize_SizeRangeCount_100-1000": 4, "ResponseSize_SizeRangeCount_1000-10000": 14, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 120, "RequestSize_min": 89, "RequestSize_max": 2901, "RequestSize_mean": 270, "RequestSize_25th_percentile": 134, "RequestSize_median": 228, "RequestSize_75th_percentile": 342, "RequestSize_90th_percentile": 1148, "RequestSize_95th_percentile": 2024, "RequestSize_98th_percentile": 2550, "RequestSize_99th_percentile": 2725, "RequestSize_99.9th_percentile": 2883, "RequestSize_SizeRangeCount_10-100": 6, "RequestSize_SizeRangeCount_100-1000": 57, "sentBytes": 49052 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:58086", "tag.serverName": "b543b351fe95,35247,1733463601969", "tag.clusterId": "976c7c09-c307-4348-ab53-202fa67db969", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95", "regionCount": 0, "storeCount": 0, "hlogFileCount": 0, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 9223372036854775807, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733463601969, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 174351, "localBytesRead": 174351, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 103692, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-06T05:40:04,663 DEBUG [HMaster-EventLoopGroup-21-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b543b351fe95,32775,1733463601942 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:40:04,663 WARN [Time-limited test {}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.regionserver.RegionServerAbortedException): Server b543b351fe95,32775,1733463601942 aborting at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:?] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:?] at org.apache.hadoop.hbase.Abortable.abort(Abortable.java:45) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster.killAll(SingleProcessHBaseCluster.java:880) ~[test-classes/:?] at org.apache.hadoop.hbase.TestZooKeeper.after(TestZooKeeper.java:107) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: Server b543b351fe95,32775,1733463601942 aborting at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-12-06T05:40:04,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741932_1110 (size=808) 2024-12-06T05:40:04,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741932_1110 (size=808) 2024-12-06T05:40:04,672 INFO [M:1;b543b351fe95:32775 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-06T05:40:04,672 INFO [M:1;b543b351fe95:32775 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=true 2024-12-06T05:40:04,677 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b543b351fe95,35247,1733463601969' ***** 2024-12-06T05:40:04,677 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: killAll 2024-12-06T05:40:04,677 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server b543b351fe95,38779,1733463601993: killAll ***** 2024-12-06T05:40:04,677 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-06T05:40:04,677 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-06T05:40:04,677 INFO [RS:0;b543b351fe95:35247 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T05:40:04,678 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T05:40:04,678 INFO [RS:0;b543b351fe95:35247 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-06T05:40:04,678 INFO [RS:0;b543b351fe95:35247 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-06T05:40:04,678 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-06T05:40:04,678 INFO [RS:0;b543b351fe95:35247 {}] regionserver.HRegionServer(3091): Received CLOSE for 6b7648f7bc7a5892f02675835e253ce4 2024-12-06T05:40:04,678 INFO [RS:0;b543b351fe95:35247 {}] regionserver.HRegionServer(3091): Received CLOSE for 16986c34f3a71270f1d68aff722e562a 2024-12-06T05:40:04,678 INFO [RS:0;b543b351fe95:35247 {}] regionserver.HRegionServer(3091): Received CLOSE for 73b074a80469c17ad4213657d37d496f 2024-12-06T05:40:04,678 INFO [RS:0;b543b351fe95:35247 {}] regionserver.HRegionServer(956): aborting server b543b351fe95,35247,1733463601969 2024-12-06T05:40:04,678 INFO [RS:0;b543b351fe95:35247 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T05:40:04,678 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6b7648f7bc7a5892f02675835e253ce4, disabling compactions & flushes 2024-12-06T05:40:04,678 INFO [RS:0;b543b351fe95:35247 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;b543b351fe95:35247. 2024-12-06T05:40:04,678 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1733463603410.6b7648f7bc7a5892f02675835e253ce4. 2024-12-06T05:40:04,678 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1733463603410.6b7648f7bc7a5892f02675835e253ce4. 2024-12-06T05:40:04,678 DEBUG [RS:0;b543b351fe95:35247 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T05:40:04,678 DEBUG [RS:0;b543b351fe95:35247 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:40:04,678 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1733463603410.6b7648f7bc7a5892f02675835e253ce4. after waiting 0 ms 2024-12-06T05:40:04,678 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-06T05:40:04,678 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1733463603410.6b7648f7bc7a5892f02675835e253ce4. 2024-12-06T05:40:04,678 INFO [RS:0;b543b351fe95:35247 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-12-06T05:40:04,678 DEBUG [RS:0;b543b351fe95:35247 {}] regionserver.HRegionServer(1325): Online Regions={6b7648f7bc7a5892f02675835e253ce4=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1733463603410.6b7648f7bc7a5892f02675835e253ce4., 16986c34f3a71270f1d68aff722e562a=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1733463603410.16986c34f3a71270f1d68aff722e562a., 73b074a80469c17ad4213657d37d496f=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1733463603410.73b074a80469c17ad4213657d37d496f.} 2024-12-06T05:40:04,678 DEBUG [RS:0;b543b351fe95:35247 {}] regionserver.HRegionServer(1351): Waiting on 16986c34f3a71270f1d68aff722e562a, 6b7648f7bc7a5892f02675835e253ce4, 73b074a80469c17ad4213657d37d496f 2024-12-06T05:40:04,678 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-06T05:40:04,680 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 619168160 }, "NonHeapMemoryUsage": { "committed": 171180032, "init": 7667712, "max": -1, "used": 168644328 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 36454, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 1, "ProcessCallTime_num_ops": 120, "ProcessCallTime_min": 0, "ProcessCallTime_max": 15, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 1, "ProcessCallTime_75th_percentile": 3, "ProcessCallTime_90th_percentile": 6, "ProcessCallTime_95th_percentile": 6, "ProcessCallTime_98th_percentile": 13, "ProcessCallTime_99th_percentile": 15, "ProcessCallTime_99.9th_percentile": 15, "ProcessCallTime_TimeRangeCount_0-1": 32, "ProcessCallTime_TimeRangeCount_1-3": 18, "ProcessCallTime_TimeRangeCount_3-10": 10, "ProcessCallTime_TimeRangeCount_10-30": 3, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 120, "QueueCallTime_min": 0, "QueueCallTime_max": 19, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 1, "QueueCallTime_95th_percentile": 5, "QueueCallTime_98th_percentile": 15, "QueueCallTime_99th_percentile": 19, "QueueCallTime_99.9th_percentile": 19, "QueueCallTime_TimeRangeCount_0-1": 58, "QueueCallTime_TimeRangeCount_1-3": 1, "QueueCallTime_TimeRangeCount_3-10": 1, "QueueCallTime_TimeRangeCount_10-30": 3, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 120, "TotalCallTime_min": 0, "TotalCallTime_max": 32, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 3, "TotalCallTime_90th_percentile": 6, "TotalCallTime_95th_percentile": 15, "TotalCallTime_98th_percentile": 19, "TotalCallTime_99th_percentile": 32, "TotalCallTime_99.9th_percentile": 32, "TotalCallTime_TimeRangeCount_0-1": 28, "TotalCallTime_TimeRangeCount_1-3": 21, "TotalCallTime_TimeRangeCount_3-10": 8, "TotalCallTime_TimeRangeCount_10-30": 5, "TotalCallTime_TimeRangeCount_30-100": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 120, "ResponseSize_min": 0, "ResponseSize_max": 6999, "ResponseSize_mean": 837, "ResponseSize_25th_percentile": 1, "ResponseSize_median": 3, "ResponseSize_75th_percentile": 177, "ResponseSize_90th_percentile": 4326, "ResponseSize_95th_percentile": 5662, "ResponseSize_98th_percentile": 6464, "ResponseSize_99th_percentile": 6731, "ResponseSize_99.9th_percentile": 6972, "ResponseSize_SizeRangeCount_0-10": 36, "ResponseSize_SizeRangeCount_10-100": 9, "ResponseSize_SizeRangeCount_100-1000": 4, "ResponseSize_SizeRangeCount_1000-10000": 14, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 120, "RequestSize_min": 89, "RequestSize_max": 2901, "RequestSize_mean": 270, "RequestSize_25th_percentile": 134, "RequestSize_median": 228, "RequestSize_75th_percentile": 342, "RequestSize_90th_percentile": 1148, "RequestSize_95th_percentile": 2024, "RequestSize_98th_percentile": 2550, "RequestSize_99th_percentile": 2725, "RequestSize_99.9th_percentile": 2883, "RequestSize_SizeRangeCount_10-100": 6, "RequestSize_SizeRangeCount_100-1000": 57, "sentBytes": 49052 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:58086", "tag.serverName": "b543b351fe95,35247,1733463601969", "tag.clusterId": "976c7c09-c307-4348-ab53-202fa67db969", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95", "regionCount": 0, "storeCount": 0, "hlogFileCount": 0, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 9223372036854775807, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733463601969, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 174351, "localBytesRead": 174351, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 103692, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-06T05:40:04,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741912_1089 (size=118305) 2024-12-06T05:40:04,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741912_1089 (size=118305) 2024-12-06T05:40:04,681 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1733463603410.6b7648f7bc7a5892f02675835e253ce4. 2024-12-06T05:40:04,681 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6b7648f7bc7a5892f02675835e253ce4: Waiting for close lock at 1733463604678Running coprocessor pre-close hooks at 1733463604678Disabling compacts and flushes for region at 1733463604678Disabling writes for close at 1733463604678Writing region close event to WAL at 1733463604681 (+3 ms)Running coprocessor post-close hooks at 1733463604681Closed at 1733463604681 2024-12-06T05:40:04,682 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,d,1733463603410.6b7648f7bc7a5892f02675835e253ce4. 2024-12-06T05:40:04,682 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 16986c34f3a71270f1d68aff722e562a, disabling compactions & flushes 2024-12-06T05:40:04,682 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1733463603410.16986c34f3a71270f1d68aff722e562a. 2024-12-06T05:40:04,682 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1733463603410.16986c34f3a71270f1d68aff722e562a. 2024-12-06T05:40:04,682 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1733463603410.16986c34f3a71270f1d68aff722e562a. after waiting 0 ms 2024-12-06T05:40:04,682 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1733463603410.16986c34f3a71270f1d68aff722e562a. 2024-12-06T05:40:04,682 DEBUG [HMaster-EventLoopGroup-21-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b543b351fe95,32775,1733463601942 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:40:04,683 WARN [Time-limited test {}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.regionserver.RegionServerAbortedException): Server b543b351fe95,32775,1733463601942 aborting at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:?] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:?] at org.apache.hadoop.hbase.Abortable.abort(Abortable.java:45) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster.killAll(SingleProcessHBaseCluster.java:880) ~[test-classes/:?] at org.apache.hadoop.hbase.TestZooKeeper.after(TestZooKeeper.java:107) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: Server b543b351fe95,32775,1733463601942 aborting at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-12-06T05:40:04,682 WARN [Close-WAL-Writer-0 {}] wal.AsyncProtobufLogWriter(165): normal close failed, try recover java.io.IOException: Commit or complete block blk_1073741912_1089, whereas it is under recovery. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.commitOrCompleteLastBlock(BlockManager.java:1211) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.commitOrCompleteLastBlock(FSNamesystem.java:3890) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:731) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:163) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: Commit or complete block blk_1073741912_1089, whereas it is under recovery. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.commitOrCompleteLastBlock(BlockManager.java:1211) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.commitOrCompleteLastBlock(FSNamesystem.java:3890) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:731) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 6 more 2024-12-06T05:40:04,684 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,32775,1733463601942/b543b351fe95%2C32775%2C1733463601942.1733463602480 2024-12-06T05:40:04,685 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,32775,1733463601942/b543b351fe95%2C32775%2C1733463601942.1733463602480 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.recoverLease(FSNamesystem.java:2869) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.recoverLease(NameNodeRpcServer.java:872) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.recoverLease(ClientNamenodeProtocolServerSideTranslatorPB.java:834) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:951) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.recoverAndClose(FanOutOneBlockAsyncDFSOutput.java:605) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:166) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,32775,1733463601942/b543b351fe95%2C32775%2C1733463601942.1733463602480 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.recoverLease(FSNamesystem.java:2869) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.recoverLease(NameNodeRpcServer.java:872) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.recoverLease(ClientNamenodeProtocolServerSideTranslatorPB.java:834) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.recoverLease(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$recoverLease$24(ClientNamenodeProtocolTranslatorPB.java:685) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.recoverLease(ClientNamenodeProtocolTranslatorPB.java:685) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:949) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 17 more 2024-12-06T05:40:04,686 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34453/user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,32775,1733463601942/b543b351fe95%2C32775%2C1733463601942.1733463602480 2024-12-06T05:40:04,687 ERROR [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2033): Unable to recover lease after several attempts. Give up. java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,32775,1733463601942/b543b351fe95%2C32775%2C1733463601942.1733463602480 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.recoverLease(FSNamesystem.java:2869) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.recoverLease(NameNodeRpcServer.java:872) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.recoverLease(ClientNamenodeProtocolServerSideTranslatorPB.java:834) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:951) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/MasterData/WALs/b543b351fe95,32775,1733463601942/b543b351fe95%2C32775%2C1733463601942.1733463602480 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.recoverLease(FSNamesystem.java:2869) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.recoverLease(NameNodeRpcServer.java:872) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.recoverLease(ClientNamenodeProtocolServerSideTranslatorPB.java:834) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.recoverLease(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$recoverLease$24(ClientNamenodeProtocolTranslatorPB.java:685) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.recoverLease(ClientNamenodeProtocolTranslatorPB.java:685) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.recoverLease(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:949) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 16 more 2024-12-06T05:40:04,688 DEBUG [M:1;b543b351fe95:32775 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T05:40:04,688 INFO [M:1;b543b351fe95:32775 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:40:04,688 DEBUG [M:1;b543b351fe95:32775 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:40:04,688 DEBUG [M:1;b543b351fe95:32775 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T05:40:04,688 DEBUG [M:1;b543b351fe95:32775 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:40:04,693 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b543b351fe95,38779,1733463601993' ***** 2024-12-06T05:40:04,693 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1733463603410.16986c34f3a71270f1d68aff722e562a. 2024-12-06T05:40:04,693 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: killAll 2024-12-06T05:40:04,693 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 16986c34f3a71270f1d68aff722e562a: Waiting for close lock at 1733463604682Running coprocessor pre-close hooks at 1733463604682Disabling compacts and flushes for region at 1733463604682Disabling writes for close at 1733463604682Writing region close event to WAL at 1733463604693 (+11 ms)Running coprocessor post-close hooks at 1733463604693Closed at 1733463604693 2024-12-06T05:40:04,693 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server b543b351fe95,39035,1733463603336: killAll ***** 2024-12-06T05:40:04,694 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-06T05:40:04,694 ERROR [M:1;b543b351fe95:32775 {}] regionserver.HRegion(1960): Memstore data size is 104861 in region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:40:04,694 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,a,1733463603410.16986c34f3a71270f1d68aff722e562a. 2024-12-06T05:40:04,694 INFO [M:1;b543b351fe95:32775 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:40:04,694 DEBUG [M:1;b543b351fe95:32775 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733463604688Disabling compacts and flushes for region at 1733463604688Disabling writes for close at 1733463604688Writing region close event to WAL at 1733463604693 (+5 ms)Closed at 1733463604694 (+1 ms) 2024-12-06T05:40:04,694 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-06T05:40:04,694 INFO [M:1;b543b351fe95:32775 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-06T05:40:04,694 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T05:40:04,694 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 73b074a80469c17ad4213657d37d496f, disabling compactions & flushes 2024-12-06T05:40:04,694 INFO [M:1;b543b351fe95:32775 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32775 2024-12-06T05:40:04,694 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1733463603410.73b074a80469c17ad4213657d37d496f. 2024-12-06T05:40:04,694 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1733463603410.73b074a80469c17ad4213657d37d496f. 2024-12-06T05:40:04,694 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1733463603410.73b074a80469c17ad4213657d37d496f. after waiting 0 ms 2024-12-06T05:40:04,694 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1733463603410.73b074a80469c17ad4213657d37d496f. 2024-12-06T05:40:04,694 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-06T05:40:04,694 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1733463603410.73b074a80469c17ad4213657d37d496f. 2024-12-06T05:40:04,694 INFO [M:1;b543b351fe95:32775 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T05:40:04,695 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 73b074a80469c17ad4213657d37d496f: Waiting for close lock at 1733463604694Running coprocessor pre-close hooks at 1733463604694Disabling compacts and flushes for region at 1733463604694Disabling writes for close at 1733463604694Writing region close event to WAL at 1733463604694Running coprocessor post-close hooks at 1733463604694Closed at 1733463604694 2024-12-06T05:40:04,695 INFO [M:1;b543b351fe95:32775 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T05:40:04,695 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-06T05:40:04,695 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,i,1733463603410.73b074a80469c17ad4213657d37d496f. 2024-12-06T05:40:04,695 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-06T05:40:04,696 INFO [RS:1;b543b351fe95:38779 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T05:40:04,696 INFO [RS:1;b543b351fe95:38779 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-06T05:40:04,696 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T05:40:04,696 INFO [RS:1;b543b351fe95:38779 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-06T05:40:04,696 INFO [RS:1;b543b351fe95:38779 {}] regionserver.HRegionServer(3091): Received CLOSE for 5bfbbde54676defb425c75f2d409c104 2024-12-06T05:40:04,697 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 621265312 }, "NonHeapMemoryUsage": { "committed": 171245568, "init": 7667712, "max": -1, "used": 168707328 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 36454, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 1, "ProcessCallTime_num_ops": 120, "ProcessCallTime_min": 0, "ProcessCallTime_max": 15, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 1, "ProcessCallTime_75th_percentile": 3, "ProcessCallTime_90th_percentile": 6, "ProcessCallTime_95th_percentile": 6, "ProcessCallTime_98th_percentile": 13, "ProcessCallTime_99th_percentile": 15, "ProcessCallTime_99.9th_percentile": 15, "ProcessCallTime_TimeRangeCount_0-1": 32, "ProcessCallTime_TimeRangeCount_1-3": 18, "ProcessCallTime_TimeRangeCount_3-10": 10, "ProcessCallTime_TimeRangeCount_10-30": 3, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 120, "QueueCallTime_min": 0, "QueueCallTime_max": 19, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 1, "QueueCallTime_95th_percentile": 5, "QueueCallTime_98th_percentile": 15, "QueueCallTime_99th_percentile": 19, "QueueCallTime_99.9th_percentile": 19, "QueueCallTime_TimeRangeCount_0-1": 58, "QueueCallTime_TimeRangeCount_1-3": 1, "QueueCallTime_TimeRangeCount_3-10": 1, "QueueCallTime_TimeRangeCount_10-30": 3, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 120, "TotalCallTime_min": 0, "TotalCallTime_max": 32, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 3, "TotalCallTime_90th_percentile": 6, "TotalCallTime_95th_percentile": 15, "TotalCallTime_98th_percentile": 19, "TotalCallTime_99th_percentile": 32, "TotalCallTime_99.9th_percentile": 32, "TotalCallTime_TimeRangeCount_0-1": 28, "TotalCallTime_TimeRangeCount_1-3": 21, "TotalCallTime_TimeRangeCount_3-10": 8, "TotalCallTime_TimeRangeCount_10-30": 5, "TotalCallTime_TimeRangeCount_30-100": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 120, "ResponseSize_min": 0, "ResponseSize_max": 6999, "ResponseSize_mean": 837, "ResponseSize_25th_percentile": 1, "ResponseSize_median": 3, "ResponseSize_75th_percentile": 177, "ResponseSize_90th_percentile": 4326, "ResponseSize_95th_percentile": 5662, "ResponseSize_98th_percentile": 6464, "ResponseSize_99th_percentile": 6731, "ResponseSize_99.9th_percentile": 6972, "ResponseSize_SizeRangeCount_0-10": 36, "ResponseSize_SizeRangeCount_10-100": 9, "ResponseSize_SizeRangeCount_100-1000": 4, "ResponseSize_SizeRangeCount_1000-10000": 14, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 120, "RequestSize_min": 89, "RequestSize_max": 2901, "RequestSize_mean": 270, "RequestSize_25th_percentile": 134, "RequestSize_median": 228, "RequestSize_75th_percentile": 342, "RequestSize_90th_percentile": 1148, "RequestSize_95th_percentile": 2024, "RequestSize_98th_percentile": 2550, "RequestSize_99th_percentile": 2725, "RequestSize_99.9th_percentile": 2883, "RequestSize_SizeRangeCount_10-100": 6, "RequestSize_SizeRangeCount_100-1000": 57, "sentBytes": 49052 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:58086", "tag.serverName": "b543b351fe95,35247,1733463601969", "tag.clusterId": "976c7c09-c307-4348-ab53-202fa67db969", "tag.Context": "regionserver", "tag.Hostname": "b543b351fe95", "regionCount": 0, "storeCount": 0, "hlogFileCount": 0, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 9223372036854775807, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733463601969, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 174351, "localBytesRead": 174351, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 103692, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-06T05:40:04,697 INFO [RS:1;b543b351fe95:38779 {}] regionserver.HRegionServer(3091): Received CLOSE for ced409d1a7981960060534bbb519e5d7 2024-12-06T05:40:04,697 INFO [RS:1;b543b351fe95:38779 {}] regionserver.HRegionServer(3091): Received CLOSE for cc3edcebfba16936afa2293e025fd871 2024-12-06T05:40:04,697 INFO [RS:1;b543b351fe95:38779 {}] regionserver.HRegionServer(3091): Received CLOSE for 23e1669d19d0104a438a531c9f1c918d 2024-12-06T05:40:04,697 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5bfbbde54676defb425c75f2d409c104, disabling compactions & flushes 2024-12-06T05:40:04,697 INFO [RS:1;b543b351fe95:38779 {}] regionserver.HRegionServer(956): aborting server b543b351fe95,38779,1733463601993 2024-12-06T05:40:04,697 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1733463603410.5bfbbde54676defb425c75f2d409c104. 2024-12-06T05:40:04,697 INFO [RS:1;b543b351fe95:38779 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T05:40:04,697 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1733463603410.5bfbbde54676defb425c75f2d409c104. 2024-12-06T05:40:04,697 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1733463603410.5bfbbde54676defb425c75f2d409c104. after waiting 0 ms 2024-12-06T05:40:04,697 INFO [RS:1;b543b351fe95:38779 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;b543b351fe95:38779. 2024-12-06T05:40:04,697 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1733463603410.5bfbbde54676defb425c75f2d409c104. 2024-12-06T05:40:04,698 DEBUG [RS:1;b543b351fe95:38779 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T05:40:04,697 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-24-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server b543b351fe95:32775 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b543b351fe95/172.17.0.2:32775 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:40:04,698 DEBUG [RS:1;b543b351fe95:38779 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:40:04,698 INFO [RS:1;b543b351fe95:38779 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T05:40:04,698 INFO [RS:1;b543b351fe95:38779 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T05:40:04,698 INFO [RS:1;b543b351fe95:38779 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T05:40:04,698 INFO [RS:1;b543b351fe95:38779 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-06T05:40:04,698 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-24-2 {}] ipc.FailedServers(52): Added failed server with address b543b351fe95:32775 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b543b351fe95/172.17.0.2:32775 2024-12-06T05:40:04,698 WARN [Time-limited test {}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: java.net.ConnectException: Call to address=b543b351fe95:32775 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b543b351fe95/172.17.0.2:32775 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:?] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:?] at org.apache.hadoop.hbase.Abortable.abort(Abortable.java:45) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster.killAll(SingleProcessHBaseCluster.java:880) ~[test-classes/:?] at org.apache.hadoop.hbase.TestZooKeeper.after(TestZooKeeper.java:107) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=b543b351fe95:32775 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b543b351fe95/172.17.0.2:32775 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b543b351fe95/172.17.0.2:32775 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-12-06T05:40:04,699 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1733463603410.5bfbbde54676defb425c75f2d409c104. 2024-12-06T05:40:04,699 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5bfbbde54676defb425c75f2d409c104: Waiting for close lock at 1733463604697Running coprocessor pre-close hooks at 1733463604697Disabling compacts and flushes for region at 1733463604697Disabling writes for close at 1733463604697Writing region close event to WAL at 1733463604699 (+2 ms)Running coprocessor post-close hooks at 1733463604699Closed at 1733463604699 2024-12-06T05:40:04,699 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,e,1733463603410.5bfbbde54676defb425c75f2d409c104. 2024-12-06T05:40:04,699 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ced409d1a7981960060534bbb519e5d7, disabling compactions & flushes 2024-12-06T05:40:04,699 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1733463603410.ced409d1a7981960060534bbb519e5d7. 2024-12-06T05:40:04,699 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1733463603410.ced409d1a7981960060534bbb519e5d7. 2024-12-06T05:40:04,699 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1733463603410.ced409d1a7981960060534bbb519e5d7. after waiting 0 ms 2024-12-06T05:40:04,699 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1733463603410.ced409d1a7981960060534bbb519e5d7. 2024-12-06T05:40:04,705 INFO [RS:1;b543b351fe95:38779 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-12-06T05:40:04,705 DEBUG [RS:1;b543b351fe95:38779 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, ced409d1a7981960060534bbb519e5d7=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1733463603410.ced409d1a7981960060534bbb519e5d7., cc3edcebfba16936afa2293e025fd871=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1733463603410.cc3edcebfba16936afa2293e025fd871., 23e1669d19d0104a438a531c9f1c918d=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1733463603410.23e1669d19d0104a438a531c9f1c918d.} 2024-12-06T05:40:04,705 DEBUG [RS:1;b543b351fe95:38779 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 23e1669d19d0104a438a531c9f1c918d, cc3edcebfba16936afa2293e025fd871, ced409d1a7981960060534bbb519e5d7 2024-12-06T05:40:04,705 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b543b351fe95,39035,1733463603336' ***** 2024-12-06T05:40:04,705 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: killAll 2024-12-06T05:40:04,705 INFO [RS:2;b543b351fe95:39035 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T05:40:04,705 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T05:40:04,705 INFO [RS:2;b543b351fe95:39035 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-06T05:40:04,705 INFO [RS:2;b543b351fe95:39035 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-06T05:40:04,705 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.TestZooKeeper.after(TestZooKeeper.java:110) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T05:40:04,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:40:04,705 INFO [RS:2;b543b351fe95:39035 {}] regionserver.HRegionServer(3091): Received CLOSE for 8b92523d97289af37e0b4eb02850f93c 2024-12-06T05:40:04,706 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2f257e98 to 127.0.0.1:58086 2024-12-06T05:40:04,706 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-06T05:40:04,705 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T05:40:04,706 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1733463603410.ced409d1a7981960060534bbb519e5d7. 2024-12-06T05:40:04,706 INFO [RS:2;b543b351fe95:39035 {}] regionserver.HRegionServer(3091): Received CLOSE for a8d0bc8f1ee440609a75868892f81f67 2024-12-06T05:40:04,706 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ced409d1a7981960060534bbb519e5d7: Waiting for close lock at 1733463604699Running coprocessor pre-close hooks at 1733463604699Disabling compacts and flushes for region at 1733463604699Disabling writes for close at 1733463604699Writing region close event to WAL at 1733463604706 (+7 ms)Running coprocessor post-close hooks at 1733463604706Closed at 1733463604706 2024-12-06T05:40:04,706 INFO [RS:2;b543b351fe95:39035 {}] regionserver.HRegionServer(3091): Received CLOSE for c0251d107f2e4c28a8d176e8af25b695 2024-12-06T05:40:04,706 INFO [RS:2;b543b351fe95:39035 {}] regionserver.HRegionServer(3091): Received CLOSE for b20ebfffac3488c42f88544a5529f43f 2024-12-06T05:40:04,706 INFO [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T05:40:04,706 INFO [RS:2;b543b351fe95:39035 {}] regionserver.HRegionServer(956): aborting server b543b351fe95,39035,1733463603336 2024-12-06T05:40:04,706 INFO [RS:2;b543b351fe95:39035 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T05:40:04,706 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,,1733463603410.ced409d1a7981960060534bbb519e5d7. 2024-12-06T05:40:04,706 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T05:40:04,706 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T05:40:04,706 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T05:40:04,706 INFO [RS:2;b543b351fe95:39035 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;b543b351fe95:39035. 2024-12-06T05:40:04,706 DEBUG [RS:2;b543b351fe95:39035 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T05:40:04,706 DEBUG [RS:2;b543b351fe95:39035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:40:04,706 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cc3edcebfba16936afa2293e025fd871, disabling compactions & flushes 2024-12-06T05:40:04,706 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1733463603410.cc3edcebfba16936afa2293e025fd871. 2024-12-06T05:40:04,706 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1733463603410.cc3edcebfba16936afa2293e025fd871. 2024-12-06T05:40:04,707 INFO [RS:2;b543b351fe95:39035 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-12-06T05:40:04,707 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1733463603410.cc3edcebfba16936afa2293e025fd871. after waiting 0 ms 2024-12-06T05:40:04,707 DEBUG [RS:2;b543b351fe95:39035 {}] regionserver.HRegionServer(1325): Online Regions={8b92523d97289af37e0b4eb02850f93c=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1733463603410.8b92523d97289af37e0b4eb02850f93c., a8d0bc8f1ee440609a75868892f81f67=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1733463603410.a8d0bc8f1ee440609a75868892f81f67., c0251d107f2e4c28a8d176e8af25b695=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1733463603410.c0251d107f2e4c28a8d176e8af25b695., b20ebfffac3488c42f88544a5529f43f=testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1733463603410.b20ebfffac3488c42f88544a5529f43f.} 2024-12-06T05:40:04,707 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1733463603410.cc3edcebfba16936afa2293e025fd871. 2024-12-06T05:40:04,707 DEBUG [RS:2;b543b351fe95:39035 {}] regionserver.HRegionServer(1351): Waiting on 8b92523d97289af37e0b4eb02850f93c, a8d0bc8f1ee440609a75868892f81f67, b20ebfffac3488c42f88544a5529f43f, c0251d107f2e4c28a8d176e8af25b695 2024-12-06T05:40:04,707 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8b92523d97289af37e0b4eb02850f93c, disabling compactions & flushes 2024-12-06T05:40:04,707 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T05:40:04,707 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1733463603410.8b92523d97289af37e0b4eb02850f93c. 2024-12-06T05:40:04,707 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1733463603410.8b92523d97289af37e0b4eb02850f93c. 2024-12-06T05:40:04,707 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1733463603410.8b92523d97289af37e0b4eb02850f93c. after waiting 0 ms 2024-12-06T05:40:04,707 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1733463603410.8b92523d97289af37e0b4eb02850f93c. 2024-12-06T05:40:04,707 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1733463603410.cc3edcebfba16936afa2293e025fd871. 2024-12-06T05:40:04,707 ERROR [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 18827 in region hbase:meta,,1.1588230740 2024-12-06T05:40:04,707 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-06T05:40:04,707 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cc3edcebfba16936afa2293e025fd871: Waiting for close lock at 1733463604706Running coprocessor pre-close hooks at 1733463604706Disabling compacts and flushes for region at 1733463604706Disabling writes for close at 1733463604707 (+1 ms)Writing region close event to WAL at 1733463604707Running coprocessor post-close hooks at 1733463604707Closed at 1733463604707 2024-12-06T05:40:04,707 INFO [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T05:40:04,707 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733463604705Running coprocessor pre-close hooks at 1733463604705Disabling compacts and flushes for region at 1733463604705Disabling writes for close at 1733463604706 (+1 ms)Writing region close event to WAL at 1733463604707 (+1 ms)Running coprocessor post-close hooks at 1733463604707Closed at 1733463604707 2024-12-06T05:40:04,707 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,j,1733463603410.cc3edcebfba16936afa2293e025fd871. 2024-12-06T05:40:04,708 DEBUG [RS_CLOSE_META-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-06T05:40:04,708 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1733463603410.8b92523d97289af37e0b4eb02850f93c. 2024-12-06T05:40:04,708 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 23e1669d19d0104a438a531c9f1c918d, disabling compactions & flushes 2024-12-06T05:40:04,708 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8b92523d97289af37e0b4eb02850f93c: Waiting for close lock at 1733463604707Running coprocessor pre-close hooks at 1733463604707Disabling compacts and flushes for region at 1733463604707Disabling writes for close at 1733463604707Writing region close event to WAL at 1733463604707Running coprocessor post-close hooks at 1733463604707Closed at 1733463604708 (+1 ms) 2024-12-06T05:40:04,708 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1733463603410.23e1669d19d0104a438a531c9f1c918d. 2024-12-06T05:40:04,708 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1733463603410.23e1669d19d0104a438a531c9f1c918d. 2024-12-06T05:40:04,708 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,g,1733463603410.8b92523d97289af37e0b4eb02850f93c. 2024-12-06T05:40:04,708 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1733463603410.23e1669d19d0104a438a531c9f1c918d. after waiting 0 ms 2024-12-06T05:40:04,708 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1733463603410.23e1669d19d0104a438a531c9f1c918d. 2024-12-06T05:40:04,708 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a8d0bc8f1ee440609a75868892f81f67, disabling compactions & flushes 2024-12-06T05:40:04,708 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1733463603410.a8d0bc8f1ee440609a75868892f81f67. 2024-12-06T05:40:04,708 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1733463603410.a8d0bc8f1ee440609a75868892f81f67. 2024-12-06T05:40:04,708 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1733463603410.a8d0bc8f1ee440609a75868892f81f67. after waiting 0 ms 2024-12-06T05:40:04,708 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1733463603410.a8d0bc8f1ee440609a75868892f81f67. 2024-12-06T05:40:04,708 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1733463603410.23e1669d19d0104a438a531c9f1c918d. 2024-12-06T05:40:04,708 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 23e1669d19d0104a438a531c9f1c918d: Waiting for close lock at 1733463604708Running coprocessor pre-close hooks at 1733463604708Disabling compacts and flushes for region at 1733463604708Disabling writes for close at 1733463604708Writing region close event to WAL at 1733463604708Running coprocessor post-close hooks at 1733463604708Closed at 1733463604708 2024-12-06T05:40:04,708 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,b,1733463603410.23e1669d19d0104a438a531c9f1c918d. 2024-12-06T05:40:04,708 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1733463603410.a8d0bc8f1ee440609a75868892f81f67. 2024-12-06T05:40:04,708 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a8d0bc8f1ee440609a75868892f81f67: Waiting for close lock at 1733463604708Running coprocessor pre-close hooks at 1733463604708Disabling compacts and flushes for region at 1733463604708Disabling writes for close at 1733463604708Writing region close event to WAL at 1733463604708Running coprocessor post-close hooks at 1733463604708Closed at 1733463604708 2024-12-06T05:40:04,708 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,h,1733463603410.a8d0bc8f1ee440609a75868892f81f67. 2024-12-06T05:40:04,709 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c0251d107f2e4c28a8d176e8af25b695, disabling compactions & flushes 2024-12-06T05:40:04,709 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1733463603410.c0251d107f2e4c28a8d176e8af25b695. 2024-12-06T05:40:04,709 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1733463603410.c0251d107f2e4c28a8d176e8af25b695. 2024-12-06T05:40:04,709 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1733463603410.c0251d107f2e4c28a8d176e8af25b695. after waiting 0 ms 2024-12-06T05:40:04,709 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1733463603410.c0251d107f2e4c28a8d176e8af25b695. 2024-12-06T05:40:04,709 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1733463603410.c0251d107f2e4c28a8d176e8af25b695. 2024-12-06T05:40:04,709 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c0251d107f2e4c28a8d176e8af25b695: Waiting for close lock at 1733463604708Running coprocessor pre-close hooks at 1733463604708Disabling compacts and flushes for region at 1733463604708Disabling writes for close at 1733463604709 (+1 ms)Writing region close event to WAL at 1733463604709Running coprocessor post-close hooks at 1733463604709Closed at 1733463604709 2024-12-06T05:40:04,709 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,c,1733463603410.c0251d107f2e4c28a8d176e8af25b695. 2024-12-06T05:40:04,709 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b20ebfffac3488c42f88544a5529f43f, disabling compactions & flushes 2024-12-06T05:40:04,709 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1733463603410.b20ebfffac3488c42f88544a5529f43f. 2024-12-06T05:40:04,709 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1733463603410.b20ebfffac3488c42f88544a5529f43f. 2024-12-06T05:40:04,709 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1733463603410.b20ebfffac3488c42f88544a5529f43f. after waiting 0 ms 2024-12-06T05:40:04,709 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1733463603410.b20ebfffac3488c42f88544a5529f43f. 2024-12-06T05:40:04,725 INFO [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1733463603410.b20ebfffac3488c42f88544a5529f43f. 2024-12-06T05:40:04,725 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b20ebfffac3488c42f88544a5529f43f: Waiting for close lock at 1733463604709Running coprocessor pre-close hooks at 1733463604709Disabling compacts and flushes for region at 1733463604709Disabling writes for close at 1733463604709Writing region close event to WAL at 1733463604725 (+16 ms)Running coprocessor post-close hooks at 1733463604725Closed at 1733463604725 2024-12-06T05:40:04,725 DEBUG [RS_CLOSE_REGION-regionserver/b543b351fe95:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRegionAssignmentAfterMasterRecoveryDueToZKExpiry,f,1733463603410.b20ebfffac3488c42f88544a5529f43f. 2024-12-06T05:40:04,743 INFO [regionserver/b543b351fe95:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-06T05:40:04,743 INFO [regionserver/b543b351fe95:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-06T05:40:04,748 INFO [regionserver/b543b351fe95:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T05:40:04,753 INFO [regionserver/b543b351fe95:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T05:40:04,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:40:04,760 INFO [M:0;b543b351fe95:46477 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T05:40:04,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46477-0x100ae0a5c730019, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:40:04,785 INFO [regionserver/b543b351fe95:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T05:40:04,879 INFO [RS:0;b543b351fe95:35247 {}] regionserver.HRegionServer(976): stopping server b543b351fe95,35247,1733463601969; all regions closed. 2024-12-06T05:40:04,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741916_1093 (size=1305) 2024-12-06T05:40:04,881 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/dd0513d8-0f16-e8c0-dcff-1777aa0bf279/WALs/b543b351fe95,35247,1733463601969/b543b351fe95%2C35247%2C1733463601969.1733463602783 not finished, retry = 0 2024-12-06T05:40:04,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741916_1093 (size=1305) 2024-12-06T05:40:04,905 INFO [RS:1;b543b351fe95:38779 {}] regionserver.HRegionServer(976): stopping server b543b351fe95,38779,1733463601993; all regions closed. 2024-12-06T05:40:04,907 INFO [RS:2;b543b351fe95:39035 {}] regionserver.HRegionServer(976): stopping server b543b351fe95,39035,1733463603336; all regions closed. 2024-12-06T05:40:04,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741917_1094 (size=20589) 2024-12-06T05:40:04,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741917_1094 (size=20589) 2024-12-06T05:40:04,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741927_1104 (size=1709) 2024-12-06T05:40:04,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741927_1104 (size=1709) 2024-12-06T05:40:04,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741915_1092 (size=1708) 2024-12-06T05:40:04,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39561 is added to blk_1073741915_1092 (size=1708) 2024-12-06T05:40:04,911 DEBUG [RS:1;b543b351fe95:38779 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:40:04,911 INFO [RS:1;b543b351fe95:38779 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T05:40:04,911 INFO [RS:1;b543b351fe95:38779 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T05:40:04,912 INFO [RS:1;b543b351fe95:38779 {}] hbase.ChoreService(370): Chore service for: regionserver/b543b351fe95:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-06T05:40:04,912 INFO [RS:1;b543b351fe95:38779 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T05:40:04,912 INFO [regionserver/b543b351fe95:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T05:40:04,912 INFO [RS:1;b543b351fe95:38779 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38779 2024-12-06T05:40:04,912 DEBUG [RS:2;b543b351fe95:39035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:40:04,912 INFO [RS:2;b543b351fe95:39035 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T05:40:04,912 INFO [RS:2;b543b351fe95:39035 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T05:40:04,912 INFO [RS:2;b543b351fe95:39035 {}] hbase.ChoreService(370): Chore service for: regionserver/b543b351fe95:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-06T05:40:04,913 INFO [RS:2;b543b351fe95:39035 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T05:40:04,913 INFO [RS:2;b543b351fe95:39035 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T05:40:04,913 INFO [regionserver/b543b351fe95:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T05:40:04,913 INFO [RS:2;b543b351fe95:39035 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T05:40:04,913 INFO [RS:2;b543b351fe95:39035 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T05:40:04,913 INFO [RS:2;b543b351fe95:39035 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39035 2024-12-06T05:40:04,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x100ae0a5c73001c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b543b351fe95,38779,1733463601993 2024-12-06T05:40:04,913 INFO [RS:1;b543b351fe95:38779 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T05:40:04,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39035-0x100ae0a5c73001e, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b543b351fe95,39035,1733463603336 2024-12-06T05:40:04,914 INFO [RS:2;b543b351fe95:39035 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T05:40:04,983 DEBUG [RS:0;b543b351fe95:35247 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:40:04,983 INFO [RS:0;b543b351fe95:35247 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T05:40:04,983 INFO [RS:0;b543b351fe95:35247 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T05:40:04,983 INFO [RS:0;b543b351fe95:35247 {}] hbase.ChoreService(370): Chore service for: regionserver/b543b351fe95:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-06T05:40:04,983 INFO [RS:0;b543b351fe95:35247 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T05:40:04,983 INFO [RS:0;b543b351fe95:35247 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T05:40:04,983 INFO [RS:0;b543b351fe95:35247 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T05:40:04,983 INFO [RS:0;b543b351fe95:35247 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T05:40:04,983 INFO [regionserver/b543b351fe95:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T05:40:04,984 INFO [RS:0;b543b351fe95:35247 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35247 2024-12-06T05:40:04,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x100ae0a5c73001b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b543b351fe95,35247,1733463601969 2024-12-06T05:40:04,985 INFO [RS:0;b543b351fe95:35247 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T05:40:05,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x100ae0a5c73001c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:40:05,014 INFO [RS:1;b543b351fe95:38779 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T05:40:05,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x100ae0a5c73001c, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:40:05,014 INFO [RS:1;b543b351fe95:38779 {}] regionserver.HRegionServer(1031): Exiting; stopping=b543b351fe95,38779,1733463601993; zookeeper connection closed. 2024-12-06T05:40:05,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39035-0x100ae0a5c73001e, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:40:05,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39035-0x100ae0a5c73001e, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:40:05,014 INFO [RS:2;b543b351fe95:39035 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T05:40:05,015 INFO [RS:2;b543b351fe95:39035 {}] regionserver.HRegionServer(1031): Exiting; stopping=b543b351fe95,39035,1733463603336; zookeeper connection closed. 2024-12-06T05:40:05,015 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@77eee982 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@77eee982 2024-12-06T05:40:05,015 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4fba15a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4fba15a 2024-12-06T05:40:05,088 INFO [RS:0;b543b351fe95:35247 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T05:40:05,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x100ae0a5c73001b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:40:05,088 INFO [RS:0;b543b351fe95:35247 {}] regionserver.HRegionServer(1031): Exiting; stopping=b543b351fe95,35247,1733463601969; zookeeper connection closed. 2024-12-06T05:40:05,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35247-0x100ae0a5c73001b, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:40:05,088 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5d31e267 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5d31e267 2024-12-06T05:40:05,089 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 2 master(s) and 3 regionserver(s) complete 2024-12-06T05:40:05,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c730018, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:40:05,191 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:40:05,196 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=testing utility connecting to ZooKeeper ensemble=127.0.0.1:58086 2024-12-06T05:40:05,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility0x0, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:40:05,200 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): testing utility-0x100ae0a5c73001f connected 2024-12-06T05:40:05,201 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-06T05:40:05,202 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/meta-region-server 2024-12-06T05:40:05,203 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/rs 2024-12-06T05:40:05,203 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/splitWAL 2024-12-06T05:40:05,204 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters 2024-12-06T05:40:05,204 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc 2024-12-06T05:40:05,204 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/draining 2024-12-06T05:40:05,205 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/master-maintenance 2024-12-06T05:40:05,205 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot 2024-12-06T05:40:05,205 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/hbaseid 2024-12-06T05:40:05,206 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/table 2024-12-06T05:40:05,207 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc/abort 2024-12-06T05:40:05,207 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc/acquired 2024-12-06T05:40:05,207 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/flush-table-proc/reached 2024-12-06T05:40:05,208 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot/abort 2024-12-06T05:40:05,208 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot/acquired 2024-12-06T05:40:05,209 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Set watcher on existing znode=/hbase/online-snapshot/reached 2024-12-06T05:40:05,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/reached 2024-12-06T05:40:05,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/online-snapshot 2024-12-06T05:40:05,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/acquired 2024-12-06T05:40:05,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/abort 2024-12-06T05:40:05,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/reached 2024-12-06T05:40:05,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/flush-table-proc 2024-12-06T05:40:05,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/acquired 2024-12-06T05:40:05,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/abort 2024-12-06T05:40:05,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/table 2024-12-06T05:40:05,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:40:05,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/hbaseid 2024-12-06T05:40:05,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot 2024-12-06T05:40:05,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master-maintenance 2024-12-06T05:40:05,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/draining 2024-12-06T05:40:05,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc 2024-12-06T05:40:05,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:40:05,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/splitWAL 2024-12-06T05:40:05,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs 2024-12-06T05:40:05,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:40:05,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T05:40:05,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase 2024-12-06T05:40:05,221 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestZooKeeper#testRegionAssignmentAfterMasterRecoveryDueToZKExpiry Thread=422 (was 390) Potentially hanging thread: HMaster-EventLoopGroup-21-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.10@localhost:34453 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.9@localhost:34453 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (268780456) connection to localhost/127.0.0.1:34453 from jenkins.hfs.9 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: master:store-WAL-Roller java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:180) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-24-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-24-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (268780456) connection to localhost/127.0.0.1:34453 from jenkins.hfs.10 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-23-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-22-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1471901137_22 at /127.0.0.1:58202 [Waiting for operation #66] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-24-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-21-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-21-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/b543b351fe95:0:becomeActiveMaster java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:166) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:82) app//org.apache.hadoop.hbase.master.region.MasterRegion.replayWALs(MasterRegion.java:323) app//org.apache.hadoop.hbase.master.region.MasterRegion.open(MasterRegion.java:280) app//org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:449) app//org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) app//org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) app//org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) app//org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) app//org.apache.hadoop.hbase.master.HMaster$$Lambda$379/0x00007fef24930000.run(Unknown Source) app//org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) app//org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$360/0x00007fef248de4b8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-23-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-23-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.11@localhost:34453 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (268780456) connection to localhost/127.0.0.1:34453 from jenkins.hfs.11 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-22-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1471901137_22 at /127.0.0.1:58180 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-20-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=740 (was 685) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=430 (was 467), ProcessCount=11 (was 11), AvailableMemoryMB=4357 (was 4422) 2024-12-06T05:40:05,221 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-06T05:40:05,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x100ae0a5c73001f, quorum=127.0.0.1:58086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:40:05,331 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2dc9b531{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T05:40:05,336 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43302dd2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-06T05:40:05,336 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-06T05:40:05,336 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ac046fe{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-06T05:40:05,336 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@204c24bc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/hadoop.log.dir/,STOPPED} 2024-12-06T05:40:05,339 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-06T05:40:05,339 WARN [BP-1951228416-172.17.0.2-1733463572602 heartbeating to localhost/127.0.0.1:34453 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-06T05:40:05,339 WARN [BP-1951228416-172.17.0.2-1733463572602 heartbeating to localhost/127.0.0.1:34453 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1951228416-172.17.0.2-1733463572602 (Datanode Uuid e1e65c20-ad02-4f59-b331-e60f89cea478) service to localhost/127.0.0.1:34453 2024-12-06T05:40:05,339 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-06T05:40:05,340 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/cluster_05577cd5-3697-aeeb-f42f-dcca3f191df1/data/data3/current/BP-1951228416-172.17.0.2-1733463572602 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T05:40:05,340 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/cluster_05577cd5-3697-aeeb-f42f-dcca3f191df1/data/data4/current/BP-1951228416-172.17.0.2-1733463572602 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T05:40:05,340 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-06T05:40:05,342 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4a8ffa8b{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T05:40:05,342 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@aedc167{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-06T05:40:05,342 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-06T05:40:05,342 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@599e8739{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-06T05:40:05,343 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5e03d9d9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/hadoop.log.dir/,STOPPED} 2024-12-06T05:40:05,343 WARN [BP-1951228416-172.17.0.2-1733463572602 heartbeating to localhost/127.0.0.1:34453 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-06T05:40:05,343 WARN [BP-1951228416-172.17.0.2-1733463572602 heartbeating to localhost/127.0.0.1:34453 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1951228416-172.17.0.2-1733463572602 (Datanode Uuid ffe5ff56-7611-4f34-97e0-03f4d3941f40) service to localhost/127.0.0.1:34453 2024-12-06T05:40:05,343 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-06T05:40:05,343 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-06T05:40:05,344 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/cluster_05577cd5-3697-aeeb-f42f-dcca3f191df1/data/data1/current/BP-1951228416-172.17.0.2-1733463572602 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T05:40:05,344 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/cluster_05577cd5-3697-aeeb-f42f-dcca3f191df1/data/data2/current/BP-1951228416-172.17.0.2-1733463572602 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T05:40:05,344 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-06T05:40:05,350 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@8f70d69{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-06T05:40:05,350 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@62f5f276{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-06T05:40:05,350 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-06T05:40:05,350 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@246c61b7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-06T05:40:05,351 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5945132b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df3e600c-5a81-fc82-798b-72bae332566d/hadoop.log.dir/,STOPPED} 2024-12-06T05:40:05,360 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-06T05:40:05,396 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down