2024-11-08 17:24:44,739 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@17bffc17 2024-11-08 17:24:44,756 main DEBUG Took 0.015174 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-08 17:24:44,756 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-08 17:24:44,756 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-08 17:24:44,757 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-08 17:24:44,758 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 17:24:44,764 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-08 17:24:44,776 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 17:24:44,777 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 17:24:44,778 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 17:24:44,778 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 17:24:44,779 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 17:24:44,779 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 17:24:44,780 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 17:24:44,781 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 17:24:44,781 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 17:24:44,782 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 17:24:44,782 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 17:24:44,783 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 17:24:44,783 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 17:24:44,784 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 17:24:44,784 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 17:24:44,784 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 17:24:44,785 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 17:24:44,785 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 17:24:44,786 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 17:24:44,786 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 17:24:44,786 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 17:24:44,787 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 17:24:44,787 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 17:24:44,787 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 17:24:44,788 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 17:24:44,788 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-08 17:24:44,789 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 17:24:44,791 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-08 17:24:44,792 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-08 17:24:44,793 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-08 17:24:44,794 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-08 17:24:44,794 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-08 17:24:44,802 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-08 17:24:44,805 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-08 17:24:44,807 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-08 17:24:44,807 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-08 17:24:44,807 main DEBUG createAppenders(={Console}) 2024-11-08 17:24:44,808 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@17bffc17 initialized 2024-11-08 17:24:44,808 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@17bffc17 2024-11-08 17:24:44,809 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@17bffc17 OK. 2024-11-08 17:24:44,809 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-08 17:24:44,810 main DEBUG OutputStream closed 2024-11-08 17:24:44,810 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-08 17:24:44,810 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-08 17:24:44,810 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@52af26ee OK 2024-11-08 17:24:44,878 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-08 17:24:44,880 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-08 17:24:44,880 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-08 17:24:44,881 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-08 17:24:44,882 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-08 17:24:44,882 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-08 17:24:44,882 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-08 17:24:44,883 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-08 17:24:44,883 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-08 17:24:44,883 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-08 17:24:44,883 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-08 17:24:44,884 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-08 17:24:44,884 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-08 17:24:44,884 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-08 17:24:44,885 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-08 17:24:44,885 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-08 17:24:44,885 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-08 17:24:44,886 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-08 17:24:44,888 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08 17:24:44,888 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@528c868) with optional ClassLoader: null 2024-11-08 17:24:44,888 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-08 17:24:44,889 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@528c868] started OK. 2024-11-08T17:24:44,901 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.regionreplication.TestRegionReplicationSinkCallbackAndFlushConcurrently timeout: 13 mins 2024-11-08 17:24:44,904 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-08 17:24:44,904 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08T17:24:45,135 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5 2024-11-08T17:24:45,153 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=2, rsPorts=, rsClass=class org.apache.hadoop.hbase.regionserver.regionreplication.TestRegionReplicationSinkCallbackAndFlushConcurrently$RSForTest, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-08T17:24:45,170 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/cluster_47f703ef-7c96-6a3c-613d-f81e69896da5, deleteOnExit=true 2024-11-08T17:24:45,171 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-08T17:24:45,172 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/test.cache.data in system properties and HBase conf 2024-11-08T17:24:45,173 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/hadoop.tmp.dir in system properties and HBase conf 2024-11-08T17:24:45,174 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/hadoop.log.dir in system properties and HBase conf 2024-11-08T17:24:45,174 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-08T17:24:45,175 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-08T17:24:45,175 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-08T17:24:45,276 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-08T17:24:45,361 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T17:24:45,365 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-08T17:24:45,366 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-08T17:24:45,367 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-08T17:24:45,367 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T17:24:45,368 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-08T17:24:45,368 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-08T17:24:45,369 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T17:24:45,369 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T17:24:45,370 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-08T17:24:45,370 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/nfs.dump.dir in system properties and HBase conf 2024-11-08T17:24:45,371 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/java.io.tmpdir in system properties and HBase conf 2024-11-08T17:24:45,371 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T17:24:45,371 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-08T17:24:45,372 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-08T17:24:46,487 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-08T17:24:46,566 INFO [Time-limited test {}] log.Log(170): Logging initialized @2495ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-08T17:24:46,636 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T17:24:46,689 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T17:24:46,707 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T17:24:46,708 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T17:24:46,709 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-08T17:24:46,718 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T17:24:46,721 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7fb248e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/hadoop.log.dir/,AVAILABLE} 2024-11-08T17:24:46,722 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6cba34ce{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T17:24:46,884 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@661e928b{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/java.io.tmpdir/jetty-localhost-41097-hadoop-hdfs-3_4_1-tests_jar-_-any-14522760970694850959/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T17:24:46,890 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5c5f3317{HTTP/1.1, (http/1.1)}{localhost:41097} 2024-11-08T17:24:46,890 INFO [Time-limited test {}] server.Server(415): Started @2819ms 2024-11-08T17:24:47,422 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T17:24:47,429 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T17:24:47,430 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T17:24:47,430 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T17:24:47,430 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-08T17:24:47,431 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@290bd3b8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/hadoop.log.dir/,AVAILABLE} 2024-11-08T17:24:47,432 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2766f403{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T17:24:47,528 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5d9277a7{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/java.io.tmpdir/jetty-localhost-37607-hadoop-hdfs-3_4_1-tests_jar-_-any-3416040116846847725/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T17:24:47,529 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@ea51324{HTTP/1.1, (http/1.1)}{localhost:37607} 2024-11-08T17:24:47,529 INFO [Time-limited test {}] server.Server(415): Started @3459ms 2024-11-08T17:24:47,577 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-08T17:24:48,585 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/cluster_47f703ef-7c96-6a3c-613d-f81e69896da5/data/data1/current/BP-2061079274-172.17.0.2-1731086685883/current, will proceed with Du for space computation calculation, 2024-11-08T17:24:48,585 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/cluster_47f703ef-7c96-6a3c-613d-f81e69896da5/data/data2/current/BP-2061079274-172.17.0.2-1731086685883/current, will proceed with Du for space computation calculation, 2024-11-08T17:24:48,611 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-08T17:24:48,654 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x392619ab29fe331d with lease ID 0xe5f827887922dc3d: Processing first storage report for DS-ce08f761-01e7-4b10-b642-f5a4cf38a3ec from datanode DatanodeRegistration(127.0.0.1:40491, datanodeUuid=62b921fa-478d-4fc2-96c8-c400c50c05d9, infoPort=34955, infoSecurePort=0, ipcPort=33089, storageInfo=lv=-57;cid=testClusterID;nsid=950516654;c=1731086685883) 2024-11-08T17:24:48,655 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x392619ab29fe331d with lease ID 0xe5f827887922dc3d: from storage DS-ce08f761-01e7-4b10-b642-f5a4cf38a3ec node DatanodeRegistration(127.0.0.1:40491, datanodeUuid=62b921fa-478d-4fc2-96c8-c400c50c05d9, infoPort=34955, infoSecurePort=0, ipcPort=33089, storageInfo=lv=-57;cid=testClusterID;nsid=950516654;c=1731086685883), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-08T17:24:48,656 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x392619ab29fe331d with lease ID 0xe5f827887922dc3d: Processing first storage report for DS-71495306-5fd0-400b-8b60-5707108ac62a from datanode DatanodeRegistration(127.0.0.1:40491, datanodeUuid=62b921fa-478d-4fc2-96c8-c400c50c05d9, infoPort=34955, infoSecurePort=0, ipcPort=33089, storageInfo=lv=-57;cid=testClusterID;nsid=950516654;c=1731086685883) 2024-11-08T17:24:48,656 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x392619ab29fe331d with lease ID 0xe5f827887922dc3d: from storage DS-71495306-5fd0-400b-8b60-5707108ac62a node DatanodeRegistration(127.0.0.1:40491, datanodeUuid=62b921fa-478d-4fc2-96c8-c400c50c05d9, infoPort=34955, infoSecurePort=0, ipcPort=33089, storageInfo=lv=-57;cid=testClusterID;nsid=950516654;c=1731086685883), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-08T17:24:48,660 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5 2024-11-08T17:24:48,729 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/cluster_47f703ef-7c96-6a3c-613d-f81e69896da5/zookeeper_0, clientPort=57573, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/cluster_47f703ef-7c96-6a3c-613d-f81e69896da5/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/cluster_47f703ef-7c96-6a3c-613d-f81e69896da5/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-08T17:24:48,740 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57573 2024-11-08T17:24:48,753 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T17:24:48,757 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T17:24:48,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40491 is added to blk_1073741825_1001 (size=7) 2024-11-08T17:24:49,373 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f with version=8 2024-11-08T17:24:49,374 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/hbase-staging 2024-11-08T17:24:49,446 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-08T17:24:49,721 INFO [Time-limited test {}] client.ConnectionUtils(128): master/74e38af3904c:0 server-side Connection retries=45 2024-11-08T17:24:49,729 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T17:24:49,730 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T17:24:49,734 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T17:24:49,734 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T17:24:49,734 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T17:24:49,852 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-08T17:24:49,902 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-08T17:24:49,910 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-08T17:24:49,913 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T17:24:49,935 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 34471 (auto-detected) 2024-11-08T17:24:49,936 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-08T17:24:49,951 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42999 2024-11-08T17:24:49,972 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42999 connecting to ZooKeeper ensemble=127.0.0.1:57573 2024-11-08T17:24:50,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:429990x0, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T17:24:50,106 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42999-0x1011b7a9d710000 connected 2024-11-08T17:24:50,190 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T17:24:50,193 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T17:24:50,204 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T17:24:50,208 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f, hbase.cluster.distributed=false 2024-11-08T17:24:50,228 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T17:24:50,233 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42999 2024-11-08T17:24:50,233 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42999 2024-11-08T17:24:50,234 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42999 2024-11-08T17:24:50,236 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42999 2024-11-08T17:24:50,238 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42999 2024-11-08T17:24:50,349 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/74e38af3904c:0 server-side Connection retries=45 2024-11-08T17:24:50,351 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T17:24:50,351 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T17:24:50,351 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T17:24:50,351 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T17:24:50,351 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T17:24:50,354 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T17:24:50,357 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T17:24:50,357 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39727 2024-11-08T17:24:50,359 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39727 connecting to ZooKeeper ensemble=127.0.0.1:57573 2024-11-08T17:24:50,361 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T17:24:50,363 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T17:24:50,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:397270x0, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T17:24:50,376 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:397270x0, quorum=127.0.0.1:57573, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T17:24:50,376 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39727-0x1011b7a9d710001 connected 2024-11-08T17:24:50,380 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T17:24:50,387 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T17:24:50,390 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39727-0x1011b7a9d710001, quorum=127.0.0.1:57573, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T17:24:50,395 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39727-0x1011b7a9d710001, quorum=127.0.0.1:57573, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T17:24:50,396 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39727 2024-11-08T17:24:50,397 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39727 2024-11-08T17:24:50,397 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39727 2024-11-08T17:24:50,398 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39727 2024-11-08T17:24:50,399 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39727 2024-11-08T17:24:50,417 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/74e38af3904c:0 server-side Connection retries=45 2024-11-08T17:24:50,417 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T17:24:50,418 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T17:24:50,418 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T17:24:50,418 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T17:24:50,419 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T17:24:50,419 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T17:24:50,420 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T17:24:50,421 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44929 2024-11-08T17:24:50,423 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44929 connecting to ZooKeeper ensemble=127.0.0.1:57573 2024-11-08T17:24:50,424 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T17:24:50,429 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T17:24:50,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:449290x0, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T17:24:50,449 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:449290x0, quorum=127.0.0.1:57573, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T17:24:50,449 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44929-0x1011b7a9d710002 connected 2024-11-08T17:24:50,450 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T17:24:50,450 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T17:24:50,451 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44929-0x1011b7a9d710002, quorum=127.0.0.1:57573, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T17:24:50,453 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44929-0x1011b7a9d710002, quorum=127.0.0.1:57573, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T17:24:50,453 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44929 2024-11-08T17:24:50,454 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44929 2024-11-08T17:24:50,454 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44929 2024-11-08T17:24:50,454 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44929 2024-11-08T17:24:50,455 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44929 2024-11-08T17:24:50,470 DEBUG [M:0;74e38af3904c:42999 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;74e38af3904c:42999 2024-11-08T17:24:50,471 INFO [master/74e38af3904c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/74e38af3904c,42999,1731086689575 2024-11-08T17:24:50,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T17:24:50,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x1011b7a9d710002, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T17:24:50,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39727-0x1011b7a9d710001, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T17:24:50,483 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/74e38af3904c,42999,1731086689575 2024-11-08T17:24:50,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39727-0x1011b7a9d710001, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T17:24:50,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x1011b7a9d710002, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T17:24:50,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T17:24:50,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39727-0x1011b7a9d710001, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T17:24:50,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x1011b7a9d710002, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T17:24:50,513 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T17:24:50,514 INFO [master/74e38af3904c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/74e38af3904c,42999,1731086689575 from backup master directory 2024-11-08T17:24:50,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/74e38af3904c,42999,1731086689575 2024-11-08T17:24:50,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x1011b7a9d710002, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T17:24:50,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39727-0x1011b7a9d710001, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T17:24:50,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T17:24:50,523 WARN [master/74e38af3904c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T17:24:50,524 INFO [master/74e38af3904c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=74e38af3904c,42999,1731086689575 2024-11-08T17:24:50,526 INFO [master/74e38af3904c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-08T17:24:50,527 INFO [master/74e38af3904c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-08T17:24:50,576 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/hbase.id] with ID: cfef25bf-b44d-4c85-b9fe-d57adf3cd05e 2024-11-08T17:24:50,576 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/.tmp/hbase.id 2024-11-08T17:24:50,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40491 is added to blk_1073741826_1002 (size=42) 2024-11-08T17:24:50,992 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/.tmp/hbase.id]:[hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/hbase.id] 2024-11-08T17:24:51,041 INFO [master/74e38af3904c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T17:24:51,046 INFO [master/74e38af3904c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-08T17:24:51,065 INFO [master/74e38af3904c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-08T17:24:51,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T17:24:51,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x1011b7a9d710002, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T17:24:51,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39727-0x1011b7a9d710001, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T17:24:51,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40491 is added to blk_1073741827_1003 (size=196) 2024-11-08T17:24:51,555 INFO [master/74e38af3904c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T17:24:51,557 INFO [master/74e38af3904c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-08T17:24:51,570 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T17:24:51,574 INFO [master/74e38af3904c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T17:24:51,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40491 is added to blk_1073741828_1004 (size=1189) 2024-11-08T17:24:52,023 INFO [master/74e38af3904c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/MasterData/data/master/store 2024-11-08T17:24:52,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40491 is added to blk_1073741829_1005 (size=34) 2024-11-08T17:24:52,452 INFO [master/74e38af3904c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-08T17:24:52,455 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T17:24:52,457 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T17:24:52,457 INFO [master/74e38af3904c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T17:24:52,457 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T17:24:52,458 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T17:24:52,459 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T17:24:52,459 INFO [master/74e38af3904c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T17:24:52,460 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731086692456Disabling compacts and flushes for region at 1731086692456Disabling writes for close at 1731086692458 (+2 ms)Writing region close event to WAL at 1731086692459 (+1 ms)Closed at 1731086692459 2024-11-08T17:24:52,462 WARN [master/74e38af3904c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/MasterData/data/master/store/.initializing 2024-11-08T17:24:52,462 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/MasterData/WALs/74e38af3904c,42999,1731086689575 2024-11-08T17:24:52,470 INFO [master/74e38af3904c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T17:24:52,483 INFO [master/74e38af3904c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=74e38af3904c%2C42999%2C1731086689575, suffix=, logDir=hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/MasterData/WALs/74e38af3904c,42999,1731086689575, archiveDir=hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/MasterData/oldWALs, maxLogs=10 2024-11-08T17:24:52,504 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/MasterData/WALs/74e38af3904c,42999,1731086689575/74e38af3904c%2C42999%2C1731086689575.1731086692487, exclude list is [], retry=0 2024-11-08T17:24:52,519 DEBUG [RSForTest-EventLoopGroup-4-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40491,DS-ce08f761-01e7-4b10-b642-f5a4cf38a3ec,DISK] 2024-11-08T17:24:52,522 DEBUG [RSForTest-EventLoopGroup-4-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-08T17:24:52,553 INFO [master/74e38af3904c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/MasterData/WALs/74e38af3904c,42999,1731086689575/74e38af3904c%2C42999%2C1731086689575.1731086692487 2024-11-08T17:24:52,554 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34955:34955)] 2024-11-08T17:24:52,554 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-08T17:24:52,555 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T17:24:52,557 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T17:24:52,558 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T17:24:52,592 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T17:24:52,614 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-08T17:24:52,617 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T17:24:52,619 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T17:24:52,619 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T17:24:52,622 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-08T17:24:52,623 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T17:24:52,624 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T17:24:52,624 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T17:24:52,627 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-08T17:24:52,627 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T17:24:52,628 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T17:24:52,628 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T17:24:52,631 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-08T17:24:52,631 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T17:24:52,632 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T17:24:52,632 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T17:24:52,635 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T17:24:52,636 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T17:24:52,640 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T17:24:52,641 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T17:24:52,644 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T17:24:52,648 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T17:24:52,652 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T17:24:52,654 INFO [master/74e38af3904c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58825259, jitterRate=-0.12343533337116241}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T17:24:52,662 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731086692569Initializing all the Stores at 1731086692571 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731086692571Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731086692572 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731086692572Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731086692572Cleaning up temporary data from old regions at 1731086692641 (+69 ms)Region opened successfully at 1731086692662 (+21 ms) 2024-11-08T17:24:52,663 INFO [master/74e38af3904c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-08T17:24:52,695 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a643bbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=74e38af3904c/172.17.0.2:0 2024-11-08T17:24:52,722 INFO [master/74e38af3904c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-08T17:24:52,732 INFO [master/74e38af3904c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-08T17:24:52,732 INFO [master/74e38af3904c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-08T17:24:52,735 INFO [master/74e38af3904c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T17:24:52,736 INFO [master/74e38af3904c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-08T17:24:52,740 INFO [master/74e38af3904c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-08T17:24:52,741 INFO [master/74e38af3904c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-08T17:24:52,764 INFO [master/74e38af3904c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-08T17:24:52,773 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-08T17:24:52,828 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-08T17:24:52,831 INFO [master/74e38af3904c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-08T17:24:52,833 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-08T17:24:52,838 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-08T17:24:52,841 INFO [master/74e38af3904c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-08T17:24:52,845 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-08T17:24:52,848 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-08T17:24:52,850 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-08T17:24:52,859 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-08T17:24:52,879 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-08T17:24:52,891 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-08T17:24:52,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T17:24:52,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x1011b7a9d710002, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T17:24:52,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39727-0x1011b7a9d710001, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T17:24:52,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x1011b7a9d710002, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T17:24:52,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39727-0x1011b7a9d710001, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T17:24:52,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T17:24:52,907 INFO [master/74e38af3904c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=74e38af3904c,42999,1731086689575, sessionid=0x1011b7a9d710000, setting cluster-up flag (Was=false) 2024-11-08T17:24:52,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39727-0x1011b7a9d710001, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T17:24:52,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T17:24:52,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x1011b7a9d710002, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T17:24:52,965 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-08T17:24:52,968 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=74e38af3904c,42999,1731086689575 2024-11-08T17:24:52,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x1011b7a9d710002, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T17:24:52,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39727-0x1011b7a9d710001, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T17:24:52,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T17:24:53,028 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-08T17:24:53,031 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=74e38af3904c,42999,1731086689575 2024-11-08T17:24:53,041 INFO [master/74e38af3904c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-08T17:24:53,060 INFO [RS:0;74e38af3904c:39727 {}] regionserver.HRegionServer(746): ClusterId : cfef25bf-b44d-4c85-b9fe-d57adf3cd05e 2024-11-08T17:24:53,060 INFO [RS:1;74e38af3904c:44929 {}] regionserver.HRegionServer(746): ClusterId : cfef25bf-b44d-4c85-b9fe-d57adf3cd05e 2024-11-08T17:24:53,063 DEBUG [RS:0;74e38af3904c:39727 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T17:24:53,063 DEBUG [RS:1;74e38af3904c:44929 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T17:24:53,082 DEBUG [RS:1;74e38af3904c:44929 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T17:24:53,082 DEBUG [RS:0;74e38af3904c:39727 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T17:24:53,082 DEBUG [RS:1;74e38af3904c:44929 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T17:24:53,082 DEBUG [RS:0;74e38af3904c:39727 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T17:24:53,092 DEBUG [RS:1;74e38af3904c:44929 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T17:24:53,092 DEBUG [RS:0;74e38af3904c:39727 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T17:24:53,093 DEBUG [RS:1;74e38af3904c:44929 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2353b1a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=74e38af3904c/172.17.0.2:0 2024-11-08T17:24:53,093 DEBUG [RS:0;74e38af3904c:39727 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a86f121, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=74e38af3904c/172.17.0.2:0 2024-11-08T17:24:53,106 DEBUG [RS:1;74e38af3904c:44929 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;74e38af3904c:44929 2024-11-08T17:24:53,107 DEBUG [RS:0;74e38af3904c:39727 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;74e38af3904c:39727 2024-11-08T17:24:53,109 INFO [RS:1;74e38af3904c:44929 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T17:24:53,109 INFO [RS:0;74e38af3904c:39727 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T17:24:53,109 INFO [RS:0;74e38af3904c:39727 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T17:24:53,109 INFO [RS:1;74e38af3904c:44929 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T17:24:53,109 DEBUG [RS:0;74e38af3904c:39727 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T17:24:53,109 DEBUG [RS:1;74e38af3904c:44929 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T17:24:53,108 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T17:24:53,111 INFO [RS:0;74e38af3904c:39727 {}] regionserver.HRegionServer(2659): reportForDuty to master=74e38af3904c,42999,1731086689575 with port=39727, startcode=1731086690313 2024-11-08T17:24:53,111 INFO [RS:1;74e38af3904c:44929 {}] regionserver.HRegionServer(2659): reportForDuty to master=74e38af3904c,42999,1731086689575 with port=44929, startcode=1731086690416 2024-11-08T17:24:53,118 INFO [master/74e38af3904c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-08T17:24:53,121 DEBUG [RS:1;74e38af3904c:44929 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T17:24:53,121 DEBUG [RS:0;74e38af3904c:39727 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T17:24:53,124 INFO [master/74e38af3904c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-08T17:24:53,130 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 74e38af3904c,42999,1731086689575 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-08T17:24:53,137 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/74e38af3904c:0, corePoolSize=5, maxPoolSize=5 2024-11-08T17:24:53,137 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/74e38af3904c:0, corePoolSize=5, maxPoolSize=5 2024-11-08T17:24:53,137 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/74e38af3904c:0, corePoolSize=5, maxPoolSize=5 2024-11-08T17:24:53,138 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/74e38af3904c:0, corePoolSize=5, maxPoolSize=5 2024-11-08T17:24:53,138 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/74e38af3904c:0, corePoolSize=10, maxPoolSize=10 2024-11-08T17:24:53,138 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/74e38af3904c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T17:24:53,138 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/74e38af3904c:0, corePoolSize=2, maxPoolSize=2 2024-11-08T17:24:53,138 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/74e38af3904c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T17:24:53,143 INFO [master/74e38af3904c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731086723143 2024-11-08T17:24:53,145 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T17:24:53,145 INFO [master/74e38af3904c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-08T17:24:53,146 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-08T17:24:53,147 INFO [master/74e38af3904c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-08T17:24:53,150 INFO [master/74e38af3904c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-08T17:24:53,150 INFO [master/74e38af3904c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-08T17:24:53,151 INFO [master/74e38af3904c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-08T17:24:53,151 INFO [master/74e38af3904c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-08T17:24:53,157 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T17:24:53,157 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T17:24:53,158 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36409, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T17:24:53,158 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37269, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T17:24:53,156 INFO [master/74e38af3904c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:53,161 INFO [master/74e38af3904c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-08T17:24:53,162 INFO [master/74e38af3904c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-08T17:24:53,163 INFO [master/74e38af3904c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-08T17:24:53,165 INFO [master/74e38af3904c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-08T17:24:53,165 INFO [master/74e38af3904c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-08T17:24:53,167 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42999 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-08T17:24:53,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40491 is added to blk_1073741831_1007 (size=1321) 2024-11-08T17:24:53,167 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/74e38af3904c:0:becomeActiveMaster-HFileCleaner.large.0-1731086693166,5,FailOnTimeoutGroup] 2024-11-08T17:24:53,168 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/74e38af3904c:0:becomeActiveMaster-HFileCleaner.small.0-1731086693167,5,FailOnTimeoutGroup] 2024-11-08T17:24:53,168 INFO [master/74e38af3904c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:53,168 INFO [master/74e38af3904c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-08T17:24:53,169 INFO [master/74e38af3904c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:53,169 INFO [master/74e38af3904c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:53,172 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42999 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 74e38af3904c,44929,1731086690416 2024-11-08T17:24:53,174 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42999 {}] master.ServerManager(517): Registering regionserver=74e38af3904c,44929,1731086690416 2024-11-08T17:24:53,181 DEBUG [RS:1;74e38af3904c:44929 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f 2024-11-08T17:24:53,181 DEBUG [RS:1;74e38af3904c:44929 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36127 2024-11-08T17:24:53,181 DEBUG [RS:1;74e38af3904c:44929 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T17:24:53,190 DEBUG [RS:0;74e38af3904c:39727 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-08T17:24:53,190 WARN [RS:0;74e38af3904c:39727 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-08T17:24:53,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T17:24:53,197 DEBUG [RS:1;74e38af3904c:44929 {}] zookeeper.ZKUtil(111): regionserver:44929-0x1011b7a9d710002, quorum=127.0.0.1:57573, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/74e38af3904c,44929,1731086690416 2024-11-08T17:24:53,197 WARN [RS:1;74e38af3904c:44929 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T17:24:53,197 INFO [RS:1;74e38af3904c:44929 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T17:24:53,197 DEBUG [RS:1;74e38af3904c:44929 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/WALs/74e38af3904c,44929,1731086690416 2024-11-08T17:24:53,199 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [74e38af3904c,44929,1731086690416] 2024-11-08T17:24:53,221 INFO [RS:1;74e38af3904c:44929 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T17:24:53,237 INFO [RS:1;74e38af3904c:44929 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T17:24:53,241 INFO [RS:1;74e38af3904c:44929 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T17:24:53,241 INFO [RS:1;74e38af3904c:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:53,242 INFO [RS:1;74e38af3904c:44929 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T17:24:53,247 INFO [RS:1;74e38af3904c:44929 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T17:24:53,248 INFO [RS:1;74e38af3904c:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:53,248 DEBUG [RS:1;74e38af3904c:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/74e38af3904c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T17:24:53,248 DEBUG [RS:1;74e38af3904c:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/74e38af3904c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T17:24:53,249 DEBUG [RS:1;74e38af3904c:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/74e38af3904c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T17:24:53,249 DEBUG [RS:1;74e38af3904c:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/74e38af3904c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T17:24:53,249 DEBUG [RS:1;74e38af3904c:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/74e38af3904c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T17:24:53,249 DEBUG [RS:1;74e38af3904c:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/74e38af3904c:0, corePoolSize=2, maxPoolSize=2 2024-11-08T17:24:53,249 DEBUG [RS:1;74e38af3904c:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/74e38af3904c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T17:24:53,249 DEBUG [RS:1;74e38af3904c:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/74e38af3904c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T17:24:53,250 DEBUG [RS:1;74e38af3904c:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/74e38af3904c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T17:24:53,250 DEBUG [RS:1;74e38af3904c:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/74e38af3904c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T17:24:53,250 DEBUG [RS:1;74e38af3904c:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/74e38af3904c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T17:24:53,250 DEBUG [RS:1;74e38af3904c:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/74e38af3904c:0, corePoolSize=3, maxPoolSize=3 2024-11-08T17:24:53,250 DEBUG [RS:1;74e38af3904c:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/74e38af3904c:0, corePoolSize=3, maxPoolSize=3 2024-11-08T17:24:53,251 INFO [RS:1;74e38af3904c:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:53,251 INFO [RS:1;74e38af3904c:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:53,251 INFO [RS:1;74e38af3904c:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:53,251 INFO [RS:1;74e38af3904c:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:53,251 INFO [RS:1;74e38af3904c:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:53,252 INFO [RS:1;74e38af3904c:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=74e38af3904c,44929,1731086690416-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T17:24:53,269 INFO [RS:1;74e38af3904c:44929 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T17:24:53,270 INFO [RS:1;74e38af3904c:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=74e38af3904c,44929,1731086690416-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:53,270 INFO [RS:1;74e38af3904c:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:53,271 INFO [RS:1;74e38af3904c:44929 {}] regionserver.Replication(171): 74e38af3904c,44929,1731086690416 started 2024-11-08T17:24:53,286 INFO [RS:1;74e38af3904c:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:53,287 INFO [RS:1;74e38af3904c:44929 {}] regionserver.HRegionServer(1482): Serving as 74e38af3904c,44929,1731086690416, RpcServer on 74e38af3904c/172.17.0.2:44929, sessionid=0x1011b7a9d710002 2024-11-08T17:24:53,287 DEBUG [RS:1;74e38af3904c:44929 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T17:24:53,287 DEBUG [RS:1;74e38af3904c:44929 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 74e38af3904c,44929,1731086690416 2024-11-08T17:24:53,288 DEBUG [RS:1;74e38af3904c:44929 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '74e38af3904c,44929,1731086690416' 2024-11-08T17:24:53,288 DEBUG [RS:1;74e38af3904c:44929 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T17:24:53,289 DEBUG [RS:1;74e38af3904c:44929 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T17:24:53,289 DEBUG [RS:1;74e38af3904c:44929 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T17:24:53,290 DEBUG [RS:1;74e38af3904c:44929 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T17:24:53,290 DEBUG [RS:1;74e38af3904c:44929 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 74e38af3904c,44929,1731086690416 2024-11-08T17:24:53,290 DEBUG [RS:1;74e38af3904c:44929 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '74e38af3904c,44929,1731086690416' 2024-11-08T17:24:53,290 DEBUG [RS:1;74e38af3904c:44929 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T17:24:53,291 DEBUG [RS:1;74e38af3904c:44929 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T17:24:53,291 DEBUG [RS:1;74e38af3904c:44929 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T17:24:53,291 INFO [RS:0;74e38af3904c:39727 {}] regionserver.HRegionServer(2659): reportForDuty to master=74e38af3904c,42999,1731086689575 with port=39727, startcode=1731086690313 2024-11-08T17:24:53,291 INFO [RS:1;74e38af3904c:44929 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T17:24:53,291 INFO [RS:1;74e38af3904c:44929 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T17:24:53,292 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42999 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 74e38af3904c,39727,1731086690313 2024-11-08T17:24:53,293 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42999 {}] master.ServerManager(517): Registering regionserver=74e38af3904c,39727,1731086690313 2024-11-08T17:24:53,296 DEBUG [RS:0;74e38af3904c:39727 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f 2024-11-08T17:24:53,296 DEBUG [RS:0;74e38af3904c:39727 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36127 2024-11-08T17:24:53,296 DEBUG [RS:0;74e38af3904c:39727 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T17:24:53,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T17:24:53,336 DEBUG [RS:0;74e38af3904c:39727 {}] zookeeper.ZKUtil(111): regionserver:39727-0x1011b7a9d710001, quorum=127.0.0.1:57573, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/74e38af3904c,39727,1731086690313 2024-11-08T17:24:53,336 WARN [RS:0;74e38af3904c:39727 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T17:24:53,336 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [74e38af3904c,39727,1731086690313] 2024-11-08T17:24:53,336 INFO [RS:0;74e38af3904c:39727 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T17:24:53,336 DEBUG [RS:0;74e38af3904c:39727 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/WALs/74e38af3904c,39727,1731086690313 2024-11-08T17:24:53,342 INFO [RS:0;74e38af3904c:39727 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T17:24:53,347 INFO [RS:0;74e38af3904c:39727 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T17:24:53,348 INFO [RS:0;74e38af3904c:39727 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T17:24:53,348 INFO [RS:0;74e38af3904c:39727 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:53,348 INFO [RS:0;74e38af3904c:39727 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T17:24:53,350 INFO [RS:0;74e38af3904c:39727 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T17:24:53,350 INFO [RS:0;74e38af3904c:39727 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:53,350 DEBUG [RS:0;74e38af3904c:39727 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/74e38af3904c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T17:24:53,351 DEBUG [RS:0;74e38af3904c:39727 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/74e38af3904c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T17:24:53,351 DEBUG [RS:0;74e38af3904c:39727 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/74e38af3904c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T17:24:53,351 DEBUG [RS:0;74e38af3904c:39727 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/74e38af3904c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T17:24:53,351 DEBUG [RS:0;74e38af3904c:39727 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/74e38af3904c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T17:24:53,351 DEBUG [RS:0;74e38af3904c:39727 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/74e38af3904c:0, corePoolSize=2, maxPoolSize=2 2024-11-08T17:24:53,351 DEBUG [RS:0;74e38af3904c:39727 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/74e38af3904c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T17:24:53,351 DEBUG [RS:0;74e38af3904c:39727 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/74e38af3904c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T17:24:53,352 DEBUG [RS:0;74e38af3904c:39727 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/74e38af3904c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T17:24:53,352 DEBUG [RS:0;74e38af3904c:39727 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/74e38af3904c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T17:24:53,352 DEBUG [RS:0;74e38af3904c:39727 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/74e38af3904c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T17:24:53,352 DEBUG [RS:0;74e38af3904c:39727 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/74e38af3904c:0, corePoolSize=3, maxPoolSize=3 2024-11-08T17:24:53,352 DEBUG [RS:0;74e38af3904c:39727 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/74e38af3904c:0, corePoolSize=3, maxPoolSize=3 2024-11-08T17:24:53,353 INFO [RS:0;74e38af3904c:39727 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:53,353 INFO [RS:0;74e38af3904c:39727 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:53,353 INFO [RS:0;74e38af3904c:39727 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:53,353 INFO [RS:0;74e38af3904c:39727 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:53,353 INFO [RS:0;74e38af3904c:39727 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:53,353 INFO [RS:0;74e38af3904c:39727 {}] hbase.ChoreService(168): Chore ScheduledChore name=74e38af3904c,39727,1731086690313-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T17:24:53,369 INFO [RS:0;74e38af3904c:39727 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T17:24:53,369 INFO [RS:0;74e38af3904c:39727 {}] hbase.ChoreService(168): Chore ScheduledChore name=74e38af3904c,39727,1731086690313-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:53,369 INFO [RS:0;74e38af3904c:39727 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:53,370 INFO [RS:0;74e38af3904c:39727 {}] regionserver.Replication(171): 74e38af3904c,39727,1731086690313 started 2024-11-08T17:24:53,385 INFO [RS:0;74e38af3904c:39727 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:53,385 INFO [RS:0;74e38af3904c:39727 {}] regionserver.HRegionServer(1482): Serving as 74e38af3904c,39727,1731086690313, RpcServer on 74e38af3904c/172.17.0.2:39727, sessionid=0x1011b7a9d710001 2024-11-08T17:24:53,385 DEBUG [RS:0;74e38af3904c:39727 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T17:24:53,385 DEBUG [RS:0;74e38af3904c:39727 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 74e38af3904c,39727,1731086690313 2024-11-08T17:24:53,385 DEBUG [RS:0;74e38af3904c:39727 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '74e38af3904c,39727,1731086690313' 2024-11-08T17:24:53,385 DEBUG [RS:0;74e38af3904c:39727 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T17:24:53,386 DEBUG [RS:0;74e38af3904c:39727 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T17:24:53,387 DEBUG [RS:0;74e38af3904c:39727 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T17:24:53,387 DEBUG [RS:0;74e38af3904c:39727 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T17:24:53,387 DEBUG [RS:0;74e38af3904c:39727 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 74e38af3904c,39727,1731086690313 2024-11-08T17:24:53,387 DEBUG [RS:0;74e38af3904c:39727 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '74e38af3904c,39727,1731086690313' 2024-11-08T17:24:53,387 DEBUG [RS:0;74e38af3904c:39727 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T17:24:53,388 DEBUG [RS:0;74e38af3904c:39727 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T17:24:53,389 DEBUG [RS:0;74e38af3904c:39727 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T17:24:53,389 INFO [RS:0;74e38af3904c:39727 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T17:24:53,389 INFO [RS:0;74e38af3904c:39727 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T17:24:53,396 INFO [RS:1;74e38af3904c:44929 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T17:24:53,399 INFO [RS:1;74e38af3904c:44929 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=74e38af3904c%2C44929%2C1731086690416, suffix=, logDir=hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/WALs/74e38af3904c,44929,1731086690416, archiveDir=hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/oldWALs, maxLogs=32 2024-11-08T17:24:53,414 DEBUG [RS:1;74e38af3904c:44929 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/WALs/74e38af3904c,44929,1731086690416/74e38af3904c%2C44929%2C1731086690416.1731086693402, exclude list is [], retry=0 2024-11-08T17:24:53,418 DEBUG [RSForTest-EventLoopGroup-4-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40491,DS-ce08f761-01e7-4b10-b642-f5a4cf38a3ec,DISK] 2024-11-08T17:24:53,422 INFO [RS:1;74e38af3904c:44929 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/WALs/74e38af3904c,44929,1731086690416/74e38af3904c%2C44929%2C1731086690416.1731086693402 2024-11-08T17:24:53,422 DEBUG [RS:1;74e38af3904c:44929 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34955:34955)] 2024-11-08T17:24:53,490 INFO [RS:0;74e38af3904c:39727 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T17:24:53,494 INFO [RS:0;74e38af3904c:39727 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=74e38af3904c%2C39727%2C1731086690313, suffix=, logDir=hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/WALs/74e38af3904c,39727,1731086690313, archiveDir=hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/oldWALs, maxLogs=32 2024-11-08T17:24:53,511 DEBUG [RS:0;74e38af3904c:39727 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/WALs/74e38af3904c,39727,1731086690313/74e38af3904c%2C39727%2C1731086690313.1731086693497, exclude list is [], retry=0 2024-11-08T17:24:53,516 DEBUG [RSForTest-EventLoopGroup-4-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40491,DS-ce08f761-01e7-4b10-b642-f5a4cf38a3ec,DISK] 2024-11-08T17:24:53,519 INFO [RS:0;74e38af3904c:39727 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/WALs/74e38af3904c,39727,1731086690313/74e38af3904c%2C39727%2C1731086690313.1731086693497 2024-11-08T17:24:53,520 DEBUG [RS:0;74e38af3904c:39727 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34955:34955)] 2024-11-08T17:24:53,570 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-08T17:24:53,571 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f 2024-11-08T17:24:53,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40491 is added to blk_1073741834_1010 (size=32) 2024-11-08T17:24:53,986 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T17:24:53,989 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T17:24:53,993 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T17:24:53,993 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T17:24:53,995 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T17:24:53,995 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-08T17:24:53,997 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T17:24:53,997 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T17:24:53,998 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T17:24:53,998 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T17:24:54,000 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T17:24:54,000 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T17:24:54,001 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T17:24:54,001 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T17:24:54,003 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T17:24:54,003 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T17:24:54,004 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T17:24:54,004 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T17:24:54,005 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/hbase/meta/1588230740 2024-11-08T17:24:54,006 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/hbase/meta/1588230740 2024-11-08T17:24:54,009 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T17:24:54,009 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T17:24:54,010 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T17:24:54,012 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T17:24:54,016 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T17:24:54,017 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62005460, jitterRate=-0.0760466456413269}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T17:24:54,018 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731086693986Initializing all the Stores at 1731086693988 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731086693988Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731086693989 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731086693989Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731086693989Cleaning up temporary data from old regions at 1731086694009 (+20 ms)Region opened successfully at 1731086694018 (+9 ms) 2024-11-08T17:24:54,018 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T17:24:54,018 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T17:24:54,019 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T17:24:54,019 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T17:24:54,019 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T17:24:54,020 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T17:24:54,020 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731086694018Disabling compacts and flushes for region at 1731086694018Disabling writes for close at 1731086694019 (+1 ms)Writing region close event to WAL at 1731086694019Closed at 1731086694020 (+1 ms) 2024-11-08T17:24:54,022 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T17:24:54,022 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-08T17:24:54,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T17:24:54,035 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T17:24:54,037 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-08T17:24:54,193 DEBUG [74e38af3904c:42999 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-08T17:24:54,202 DEBUG [74e38af3904c:42999 {}] balancer.BalancerClusterState(204): Hosts are {74e38af3904c=0} racks are {/default-rack=0} 2024-11-08T17:24:54,208 DEBUG [74e38af3904c:42999 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T17:24:54,208 DEBUG [74e38af3904c:42999 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T17:24:54,208 DEBUG [74e38af3904c:42999 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T17:24:54,208 DEBUG [74e38af3904c:42999 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T17:24:54,208 INFO [74e38af3904c:42999 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T17:24:54,208 INFO [74e38af3904c:42999 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T17:24:54,208 DEBUG [74e38af3904c:42999 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T17:24:54,214 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=74e38af3904c,44929,1731086690416 2024-11-08T17:24:54,220 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 74e38af3904c,44929,1731086690416, state=OPENING 2024-11-08T17:24:54,291 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-08T17:24:54,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T17:24:54,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39727-0x1011b7a9d710001, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T17:24:54,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x1011b7a9d710002, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T17:24:54,303 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T17:24:54,303 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T17:24:54,303 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T17:24:54,305 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T17:24:54,308 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=74e38af3904c,44929,1731086690416}] 2024-11-08T17:24:54,489 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T17:24:54,492 INFO [RSForTest-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41593, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T17:24:54,502 INFO [RS_OPEN_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T17:24:54,502 INFO [RS_OPEN_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T17:24:54,503 INFO [RS_OPEN_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T17:24:54,506 INFO [RS_OPEN_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=74e38af3904c%2C44929%2C1731086690416.meta, suffix=.meta, logDir=hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/WALs/74e38af3904c,44929,1731086690416, archiveDir=hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/oldWALs, maxLogs=32 2024-11-08T17:24:54,521 DEBUG [RS_OPEN_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/WALs/74e38af3904c,44929,1731086690416/74e38af3904c%2C44929%2C1731086690416.meta.1731086694508.meta, exclude list is [], retry=0 2024-11-08T17:24:54,525 DEBUG [RSForTest-EventLoopGroup-4-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40491,DS-ce08f761-01e7-4b10-b642-f5a4cf38a3ec,DISK] 2024-11-08T17:24:54,528 INFO [RS_OPEN_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/WALs/74e38af3904c,44929,1731086690416/74e38af3904c%2C44929%2C1731086690416.meta.1731086694508.meta 2024-11-08T17:24:54,528 DEBUG [RS_OPEN_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34955:34955)] 2024-11-08T17:24:54,529 DEBUG [RS_OPEN_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T17:24:54,531 DEBUG [RS_OPEN_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T17:24:54,533 DEBUG [RS_OPEN_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T17:24:54,538 INFO [RS_OPEN_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T17:24:54,542 DEBUG [RS_OPEN_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T17:24:54,543 DEBUG [RS_OPEN_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T17:24:54,543 DEBUG [RS_OPEN_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T17:24:54,543 DEBUG [RS_OPEN_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T17:24:54,546 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T17:24:54,548 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T17:24:54,548 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T17:24:54,549 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T17:24:54,549 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-08T17:24:54,550 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T17:24:54,550 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T17:24:54,551 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T17:24:54,551 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T17:24:54,552 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T17:24:54,552 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T17:24:54,553 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T17:24:54,553 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T17:24:54,554 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T17:24:54,555 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T17:24:54,555 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T17:24:54,555 DEBUG [RS_OPEN_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T17:24:54,557 DEBUG [RS_OPEN_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/hbase/meta/1588230740 2024-11-08T17:24:54,559 DEBUG [RS_OPEN_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/hbase/meta/1588230740 2024-11-08T17:24:54,562 DEBUG [RS_OPEN_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T17:24:54,562 DEBUG [RS_OPEN_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T17:24:54,563 DEBUG [RS_OPEN_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T17:24:54,566 DEBUG [RS_OPEN_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T17:24:54,568 INFO [RS_OPEN_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65827287, jitterRate=-0.019096985459327698}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T17:24:54,568 DEBUG [RS_OPEN_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T17:24:54,569 DEBUG [RS_OPEN_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731086694543Writing region info on filesystem at 1731086694543Initializing all the Stores at 1731086694545 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731086694545Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731086694546 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731086694546Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731086694546Cleaning up temporary data from old regions at 1731086694562 (+16 ms)Running coprocessor post-open hooks at 1731086694568 (+6 ms)Region opened successfully at 1731086694569 (+1 ms) 2024-11-08T17:24:54,576 INFO [RS_OPEN_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731086694482 2024-11-08T17:24:54,587 DEBUG [RS_OPEN_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T17:24:54,587 INFO [RS_OPEN_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T17:24:54,589 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=74e38af3904c,44929,1731086690416 2024-11-08T17:24:54,591 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 74e38af3904c,44929,1731086690416, state=OPEN 2024-11-08T17:24:54,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x1011b7a9d710002, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T17:24:54,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T17:24:54,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39727-0x1011b7a9d710001, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T17:24:54,596 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T17:24:54,596 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T17:24:54,596 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T17:24:54,597 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=74e38af3904c,44929,1731086690416 2024-11-08T17:24:54,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-08T17:24:54,604 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=74e38af3904c,44929,1731086690416 in 289 msec 2024-11-08T17:24:54,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-08T17:24:54,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 578 msec 2024-11-08T17:24:54,612 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T17:24:54,612 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-08T17:24:54,630 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T17:24:54,631 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=74e38af3904c,44929,1731086690416, seqNum=-1] 2024-11-08T17:24:54,652 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T17:24:54,654 INFO [RSForTest-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46497, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T17:24:54,674 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6030 sec 2024-11-08T17:24:54,674 INFO [master/74e38af3904c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731086694674, completionTime=-1 2024-11-08T17:24:54,677 INFO [master/74e38af3904c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-08T17:24:54,677 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T17:24:54,705 INFO [master/74e38af3904c:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T17:24:54,706 INFO [master/74e38af3904c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731086754706 2024-11-08T17:24:54,706 INFO [master/74e38af3904c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731086814706 2024-11-08T17:24:54,706 INFO [master/74e38af3904c:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 28 msec 2024-11-08T17:24:54,707 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-08T17:24:54,714 INFO [master/74e38af3904c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=74e38af3904c,42999,1731086689575-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:54,714 INFO [master/74e38af3904c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=74e38af3904c,42999,1731086689575-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:54,714 INFO [master/74e38af3904c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=74e38af3904c,42999,1731086689575-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:54,716 INFO [master/74e38af3904c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-74e38af3904c:42999, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:54,716 INFO [master/74e38af3904c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:54,717 INFO [master/74e38af3904c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:54,722 DEBUG [master/74e38af3904c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-08T17:24:54,742 INFO [master/74e38af3904c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.218sec 2024-11-08T17:24:54,744 INFO [master/74e38af3904c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-08T17:24:54,745 INFO [master/74e38af3904c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-08T17:24:54,746 INFO [master/74e38af3904c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-08T17:24:54,746 INFO [master/74e38af3904c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-08T17:24:54,746 INFO [master/74e38af3904c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-08T17:24:54,747 INFO [master/74e38af3904c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=74e38af3904c,42999,1731086689575-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T17:24:54,747 INFO [master/74e38af3904c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=74e38af3904c,42999,1731086689575-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-08T17:24:54,751 DEBUG [master/74e38af3904c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-08T17:24:54,752 INFO [master/74e38af3904c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-08T17:24:54,753 INFO [master/74e38af3904c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=74e38af3904c,42999,1731086689575-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T17:24:54,769 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65e4bee4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T17:24:54,771 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-08T17:24:54,771 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-08T17:24:54,774 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 74e38af3904c,42999,-1 for getting cluster id 2024-11-08T17:24:54,776 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T17:24:54,784 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cfef25bf-b44d-4c85-b9fe-d57adf3cd05e' 2024-11-08T17:24:54,786 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T17:24:54,786 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cfef25bf-b44d-4c85-b9fe-d57adf3cd05e" 2024-11-08T17:24:54,789 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7205b5d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T17:24:54,789 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [74e38af3904c,42999,-1] 2024-11-08T17:24:54,791 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T17:24:54,793 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T17:24:54,794 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T17:24:54,797 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dc548d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T17:24:54,797 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T17:24:54,804 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=74e38af3904c,44929,1731086690416, seqNum=-1] 2024-11-08T17:24:54,805 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T17:24:54,807 INFO [RSForTest-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T17:24:54,826 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=74e38af3904c,42999,1731086689575 2024-11-08T17:24:54,839 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.regionreplication.TestRegionReplicationSinkCallbackAndFlushConcurrently#test Thread=244, OpenFileDescriptor=465, MaxFileDescriptor=1048576, SystemLoadAverage=356, ProcessCount=11, AvailableMemoryMB=5738 2024-11-08T17:24:54,843 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T17:24:54,848 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 74e38af3904c,42999,1731086689575 2024-11-08T17:24:54,851 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1a85a820 2024-11-08T17:24:54,852 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T17:24:54,855 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53146, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T17:24:54,861 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42999 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRegionReplicationSinkSuspend', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '2'}}, {NAME => 'family_test', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T17:24:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42999 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRegionReplicationSinkSuspend 2024-11-08T17:24:54,873 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRegionReplicationSinkSuspend execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T17:24:54,875 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42999 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRegionReplicationSinkSuspend" procId is: 4 2024-11-08T17:24:54,875 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T17:24:54,893 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRegionReplicationSinkSuspend execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T17:24:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42999 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T17:24:54,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40491 is added to blk_1073741836_1012 (size=416) 2024-11-08T17:24:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42999 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T17:24:55,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42999 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T17:24:55,324 INFO [RegionOpenAndInit-testRegionReplicationSinkSuspend-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 68c48d64dbf3fd3b2d45da672d351079, NAME => 'testRegionReplicationSinkSuspend,,1731086694857.68c48d64dbf3fd3b2d45da672d351079.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionReplicationSinkSuspend', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '2', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family_test', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f 2024-11-08T17:24:55,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40491 is added to blk_1073741837_1013 (size=67) 2024-11-08T17:24:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42999 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T17:24:55,735 DEBUG [RegionOpenAndInit-testRegionReplicationSinkSuspend-pool-0 {}] regionserver.HRegion(898): Instantiated testRegionReplicationSinkSuspend,,1731086694857.68c48d64dbf3fd3b2d45da672d351079.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T17:24:55,736 DEBUG [RegionOpenAndInit-testRegionReplicationSinkSuspend-pool-0 {}] regionserver.HRegion(1722): Closing 68c48d64dbf3fd3b2d45da672d351079, disabling compactions & flushes 2024-11-08T17:24:55,736 INFO [RegionOpenAndInit-testRegionReplicationSinkSuspend-pool-0 {}] regionserver.HRegion(1755): Closing region testRegionReplicationSinkSuspend,,1731086694857.68c48d64dbf3fd3b2d45da672d351079. 2024-11-08T17:24:55,736 DEBUG [RegionOpenAndInit-testRegionReplicationSinkSuspend-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionReplicationSinkSuspend,,1731086694857.68c48d64dbf3fd3b2d45da672d351079. 2024-11-08T17:24:55,736 DEBUG [RegionOpenAndInit-testRegionReplicationSinkSuspend-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRegionReplicationSinkSuspend,,1731086694857.68c48d64dbf3fd3b2d45da672d351079. after waiting 0 ms 2024-11-08T17:24:55,737 DEBUG [RegionOpenAndInit-testRegionReplicationSinkSuspend-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRegionReplicationSinkSuspend,,1731086694857.68c48d64dbf3fd3b2d45da672d351079. 2024-11-08T17:24:55,737 INFO [RegionOpenAndInit-testRegionReplicationSinkSuspend-pool-0 {}] regionserver.HRegion(1973): Closed testRegionReplicationSinkSuspend,,1731086694857.68c48d64dbf3fd3b2d45da672d351079. 2024-11-08T17:24:55,737 DEBUG [RegionOpenAndInit-testRegionReplicationSinkSuspend-pool-0 {}] regionserver.HRegion(1676): Region close journal for 68c48d64dbf3fd3b2d45da672d351079: Waiting for close lock at 1731086695736Disabling compacts and flushes for region at 1731086695736Disabling writes for close at 1731086695736Writing region close event to WAL at 1731086695737 (+1 ms)Closed at 1731086695737 2024-11-08T17:24:55,741 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRegionReplicationSinkSuspend execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T17:24:55,745 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":5,"row":"testRegionReplicationSinkSuspend,,1731086694857.68c48d64dbf3fd3b2d45da672d351079.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731086695741"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731086695741"},{"qualifier":"server_0001","vlen":0,"tag":[],"timestamp":"1731086695741"},{"qualifier":"serverstartcode_0001","vlen":0,"tag":[],"timestamp":"1731086695741"}]},"ts":"1731086695741"} 2024-11-08T17:24:55,751 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T17:24:55,752 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRegionReplicationSinkSuspend execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T17:24:55,755 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRegionReplicationSinkSuspend","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731086695752"}]},"ts":"1731086695752"} 2024-11-08T17:24:55,759 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRegionReplicationSinkSuspend, state=ENABLING in hbase:meta 2024-11-08T17:24:55,760 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {74e38af3904c=0} racks are {/default-rack=0} 2024-11-08T17:24:55,762 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 1 regions 2024-11-08T17:24:55,762 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T17:24:55,762 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T17:24:55,762 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T17:24:55,762 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T17:24:55,762 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T17:24:55,762 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-08T17:24:55,764 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionReplicationSinkSuspend, region=68c48d64dbf3fd3b2d45da672d351079, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionReplicationSinkSuspend, region=b91f008596ab08e04c70a52864038785, ASSIGN}] 2024-11-08T17:24:55,767 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionReplicationSinkSuspend, region=b91f008596ab08e04c70a52864038785, ASSIGN 2024-11-08T17:24:55,767 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRegionReplicationSinkSuspend, region=68c48d64dbf3fd3b2d45da672d351079, ASSIGN 2024-11-08T17:24:55,769 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRegionReplicationSinkSuspend, region=b91f008596ab08e04c70a52864038785, ASSIGN; state=OFFLINE, location=74e38af3904c,44929,1731086690416; forceNewPlan=false, retain=false 2024-11-08T17:24:55,769 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRegionReplicationSinkSuspend, region=68c48d64dbf3fd3b2d45da672d351079, ASSIGN; state=OFFLINE, location=74e38af3904c,39727,1731086690313; forceNewPlan=false, retain=false 2024-11-08T17:24:55,923 INFO [74e38af3904c:42999 {}] balancer.BaseLoadBalancer(388): Reassigned 2 regions. 2 retained the pre-restart assignment. 2024-11-08T17:24:55,924 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=68c48d64dbf3fd3b2d45da672d351079, regionState=OPENING, regionLocation=74e38af3904c,39727,1731086690313 2024-11-08T17:24:55,924 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=b91f008596ab08e04c70a52864038785, regionState=OPENING, regionLocation=74e38af3904c,44929,1731086690416 2024-11-08T17:24:55,929 DEBUG [RSForTest-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRegionReplicationSinkSuspend, region=68c48d64dbf3fd3b2d45da672d351079, ASSIGN because future has completed 2024-11-08T17:24:55,930 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=7, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 68c48d64dbf3fd3b2d45da672d351079, server=74e38af3904c,39727,1731086690313}] 2024-11-08T17:24:55,935 DEBUG [RSForTest-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRegionReplicationSinkSuspend, region=b91f008596ab08e04c70a52864038785, ASSIGN because future has completed 2024-11-08T17:24:55,936 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure b91f008596ab08e04c70a52864038785, server=74e38af3904c,44929,1731086690416}] 2024-11-08T17:24:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42999 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T17:24:56,086 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T17:24:56,090 INFO [RSForTest-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38453, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T17:24:56,097 INFO [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=8}] handler.AssignRegionHandler(132): Open testRegionReplicationSinkSuspend,,1731086694857_0001.b91f008596ab08e04c70a52864038785. 2024-11-08T17:24:56,097 INFO [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=7}] handler.AssignRegionHandler(132): Open testRegionReplicationSinkSuspend,,1731086694857.68c48d64dbf3fd3b2d45da672d351079. 2024-11-08T17:24:56,097 DEBUG [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=7}] regionserver.HRegion(7752): Opening region: {ENCODED => 68c48d64dbf3fd3b2d45da672d351079, NAME => 'testRegionReplicationSinkSuspend,,1731086694857.68c48d64dbf3fd3b2d45da672d351079.', STARTKEY => '', ENDKEY => ''} 2024-11-08T17:24:56,097 DEBUG [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=8}] regionserver.HRegion(7752): Opening region: {ENCODED => b91f008596ab08e04c70a52864038785, NAME => 'testRegionReplicationSinkSuspend,,1731086694857_0001.b91f008596ab08e04c70a52864038785.', STARTKEY => '', ENDKEY => '', REPLICA_ID => 1} 2024-11-08T17:24:56,098 DEBUG [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=7}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRegionReplicationSinkSuspend 68c48d64dbf3fd3b2d45da672d351079 2024-11-08T17:24:56,098 DEBUG [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=8}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRegionReplicationSinkSuspend b91f008596ab08e04c70a52864038785 2024-11-08T17:24:56,098 DEBUG [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=7}] regionserver.HRegion(898): Instantiated testRegionReplicationSinkSuspend,,1731086694857.68c48d64dbf3fd3b2d45da672d351079.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T17:24:56,098 DEBUG [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=8}] regionserver.HRegion(898): Instantiated testRegionReplicationSinkSuspend,,1731086694857_0001.b91f008596ab08e04c70a52864038785.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T17:24:56,098 DEBUG [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=7}] regionserver.HRegion(7794): checking encryption for 68c48d64dbf3fd3b2d45da672d351079 2024-11-08T17:24:56,098 DEBUG [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=7}] regionserver.HRegion(7797): checking classloading for 68c48d64dbf3fd3b2d45da672d351079 2024-11-08T17:24:56,098 DEBUG [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=8}] regionserver.HRegion(7794): checking encryption for b91f008596ab08e04c70a52864038785 2024-11-08T17:24:56,098 DEBUG [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=8}] regionserver.HRegion(7797): checking classloading for b91f008596ab08e04c70a52864038785 2024-11-08T17:24:56,099 INFO [StoreOpener-b91f008596ab08e04c70a52864038785-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family_test of region b91f008596ab08e04c70a52864038785 2024-11-08T17:24:56,100 INFO [StoreOpener-68c48d64dbf3fd3b2d45da672d351079-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family_test of region 68c48d64dbf3fd3b2d45da672d351079 2024-11-08T17:24:56,101 INFO [StoreOpener-b91f008596ab08e04c70a52864038785-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b91f008596ab08e04c70a52864038785 columnFamilyName family_test 2024-11-08T17:24:56,101 DEBUG [StoreOpener-b91f008596ab08e04c70a52864038785-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T17:24:56,102 INFO [StoreOpener-68c48d64dbf3fd3b2d45da672d351079-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 68c48d64dbf3fd3b2d45da672d351079 columnFamilyName family_test 2024-11-08T17:24:56,102 DEBUG [StoreOpener-68c48d64dbf3fd3b2d45da672d351079-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T17:24:56,102 INFO [StoreOpener-b91f008596ab08e04c70a52864038785-1 {}] regionserver.HStore(327): Store=b91f008596ab08e04c70a52864038785/family_test, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T17:24:56,103 INFO [StoreOpener-68c48d64dbf3fd3b2d45da672d351079-1 {}] regionserver.HStore(327): Store=68c48d64dbf3fd3b2d45da672d351079/family_test, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T17:24:56,103 DEBUG [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=7}] regionserver.HRegion(1038): replaying wal for 68c48d64dbf3fd3b2d45da672d351079 2024-11-08T17:24:56,104 DEBUG [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=7}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/default/testRegionReplicationSinkSuspend/68c48d64dbf3fd3b2d45da672d351079 2024-11-08T17:24:56,105 DEBUG [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=7}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/default/testRegionReplicationSinkSuspend/68c48d64dbf3fd3b2d45da672d351079 2024-11-08T17:24:56,105 INFO [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=8}] regionserver.HRegion(1114): Opened b91f008596ab08e04c70a52864038785; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61723580, jitterRate=-0.08024698495864868}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T17:24:56,105 DEBUG [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=8}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b91f008596ab08e04c70a52864038785 2024-11-08T17:24:56,105 DEBUG [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=7}] regionserver.HRegion(1048): stopping wal replay for 68c48d64dbf3fd3b2d45da672d351079 2024-11-08T17:24:56,105 DEBUG [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=7}] regionserver.HRegion(1060): Cleaning up temporary data for 68c48d64dbf3fd3b2d45da672d351079 2024-11-08T17:24:56,106 DEBUG [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=8}] regionserver.HRegion(1006): Region open journal for b91f008596ab08e04c70a52864038785: Running coprocessor pre-open hook at 1731086696098Initializing all the Stores at 1731086696098Instantiating store for column family {NAME => 'family_test', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731086696098Running coprocessor post-open hooks at 1731086696105 (+7 ms)Region opened successfully at 1731086696106 (+1 ms) 2024-11-08T17:24:56,107 INFO [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=8}] regionserver.HRegionServer(2236): Post open deploy tasks for testRegionReplicationSinkSuspend,,1731086694857_0001.b91f008596ab08e04c70a52864038785., pid=8, masterSystemTime=1731086696090 2024-11-08T17:24:56,108 DEBUG [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=7}] regionserver.HRegion(1093): writing seq id for 68c48d64dbf3fd3b2d45da672d351079 2024-11-08T17:24:56,111 DEBUG [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=8}] regionserver.HRegionServer(2266): Finished post open deploy task for testRegionReplicationSinkSuspend,,1731086694857_0001.b91f008596ab08e04c70a52864038785. 2024-11-08T17:24:56,111 INFO [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=8}] handler.AssignRegionHandler(153): Opened testRegionReplicationSinkSuspend,,1731086694857_0001.b91f008596ab08e04c70a52864038785. 2024-11-08T17:24:56,112 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=b91f008596ab08e04c70a52864038785, regionState=OPEN, openSeqNum=2, regionLocation=74e38af3904c,44929,1731086690416 2024-11-08T17:24:56,112 DEBUG [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=7}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/default/testRegionReplicationSinkSuspend/68c48d64dbf3fd3b2d45da672d351079/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T17:24:56,116 INFO [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=7}] regionserver.HRegion(1114): Opened 68c48d64dbf3fd3b2d45da672d351079; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59277399, jitterRate=-0.11669792234897614}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T17:24:56,116 DEBUG [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=7}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 68c48d64dbf3fd3b2d45da672d351079 2024-11-08T17:24:56,116 DEBUG [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=7}] regionserver.HRegion(1006): Region open journal for 68c48d64dbf3fd3b2d45da672d351079: Running coprocessor pre-open hook at 1731086696098Writing region info on filesystem at 1731086696098Initializing all the Stores at 1731086696100 (+2 ms)Instantiating store for column family {NAME => 'family_test', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731086696100Cleaning up temporary data from old regions at 1731086696105 (+5 ms)Initializaing region replication sink at 1731086696113 (+8 ms)Running coprocessor post-open hooks at 1731086696116 (+3 ms)Region opened successfully at 1731086696116 2024-11-08T17:24:56,117 DEBUG [RSForTest-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure b91f008596ab08e04c70a52864038785, server=74e38af3904c,44929,1731086690416 because future has completed 2024-11-08T17:24:56,123 DEBUG [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=7}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T17:24:56,123 DEBUG [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=7}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=74e38af3904c,44929,1731086690416, seqNum=-1] 2024-11-08T17:24:56,123 DEBUG [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=7}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T17:24:56,124 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=6 2024-11-08T17:24:56,124 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure b91f008596ab08e04c70a52864038785, server=74e38af3904c,44929,1731086690416 in 183 msec 2024-11-08T17:24:56,124 INFO [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=7}] regionserver.HRegionServer(2236): Post open deploy tasks for testRegionReplicationSinkSuspend,,1731086694857.68c48d64dbf3fd3b2d45da672d351079., pid=7, masterSystemTime=1731086696085 2024-11-08T17:24:56,125 INFO [RSForTest-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58485, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-08T17:24:56,128 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRegionReplicationSinkSuspend, region=b91f008596ab08e04c70a52864038785, ASSIGN in 360 msec 2024-11-08T17:24:56,128 DEBUG [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=7}] regionserver.HRegionServer(2266): Finished post open deploy task for testRegionReplicationSinkSuspend,,1731086694857.68c48d64dbf3fd3b2d45da672d351079. 2024-11-08T17:24:56,128 INFO [RS_OPEN_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_OPEN_REGION, pid=7}] handler.AssignRegionHandler(153): Opened testRegionReplicationSinkSuspend,,1731086694857.68c48d64dbf3fd3b2d45da672d351079. 2024-11-08T17:24:56,130 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=68c48d64dbf3fd3b2d45da672d351079, regionState=OPEN, openSeqNum=2, regionLocation=74e38af3904c,39727,1731086690313 2024-11-08T17:24:56,132 DEBUG [RSForTest-EventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRegionReplicationSinkSuspend', row='', locateType=CURRENT is [region=testRegionReplicationSinkSuspend,,1731086694857.68c48d64dbf3fd3b2d45da672d351079., hostname=null, seqNum=-1, region=testRegionReplicationSinkSuspend,,1731086694857_0001.b91f008596ab08e04c70a52864038785., hostname=74e38af3904c,44929,1731086690416, seqNum=2] 2024-11-08T17:24:56,133 WARN [RSForTest-EventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(265): Failed to locate region in 'testRegionReplicationSinkSuspend', row='', locateType=CURRENT org.apache.hadoop.hbase.HBaseIOException: No location found for 'testRegionReplicationSinkSuspend', row='', locateType=CURRENT at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.onScanNext(AsyncNonMetaRegionLocator.java:322) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator$1.onNext(AsyncNonMetaRegionLocator.java:437) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncScanSingleRegionRpcRetryingCaller.onComplete(AsyncScanSingleRegionRpcRetryingCaller.java:535) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncScanSingleRegionRpcRetryingCaller.start(AsyncScanSingleRegionRpcRetryingCaller.java:636) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCallerFactory$ScanSingleRegionCallerBuilder.start(AsyncRpcRetryingCallerFactory.java:322) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.startScan(AsyncClientScanner.java:208) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$openScanner$2(AsyncClientScanner.java:268) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:187) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T17:24:56,139 DEBUG [RSForTest-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 68c48d64dbf3fd3b2d45da672d351079, server=74e38af3904c,39727,1731086690313 because future has completed 2024-11-08T17:24:56,145 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=7, resume processing ppid=5 2024-11-08T17:24:56,145 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 68c48d64dbf3fd3b2d45da672d351079, server=74e38af3904c,39727,1731086690313 in 211 msec 2024-11-08T17:24:56,150 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-08T17:24:56,150 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRegionReplicationSinkSuspend, region=68c48d64dbf3fd3b2d45da672d351079, ASSIGN in 381 msec 2024-11-08T17:24:56,151 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRegionReplicationSinkSuspend execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T17:24:56,151 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRegionReplicationSinkSuspend","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731086696151"}]},"ts":"1731086696151"} 2024-11-08T17:24:56,154 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRegionReplicationSinkSuspend, state=ENABLED in hbase:meta 2024-11-08T17:24:56,156 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRegionReplicationSinkSuspend execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T17:24:56,159 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRegionReplicationSinkSuspend in 1.2910 sec 2024-11-08T17:24:56,254 DEBUG [RSForTest-EventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRegionReplicationSinkSuspend', row='', locateType=CURRENT is [region=testRegionReplicationSinkSuspend,,1731086694857.68c48d64dbf3fd3b2d45da672d351079., hostname=74e38af3904c,39727,1731086690313, seqNum=2, region=testRegionReplicationSinkSuspend,,1731086694857_0001.b91f008596ab08e04c70a52864038785., hostname=74e38af3904c,44929,1731086690416, seqNum=2] 2024-11-08T17:24:56,260 DEBUG [RSForTest-EventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T17:24:56,263 INFO [RSForTest-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60191, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=AdminService 2024-11-08T17:24:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] regionserver.HRegion(6778): b91f008596ab08e04c70a52864038785 : Refreshing store files to see whether we can free up memstore 2024-11-08T17:24:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42999 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T17:24:57,065 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRegionReplicationSinkSuspend completed 2024-11-08T17:24:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: Inject error! at org.apache.hadoop.hbase.regionserver.regionreplication.TestRegionReplicationSinkCallbackAndFlushConcurrently$ErrorReplayRSRpcServices.replicateToReplica(TestRegionReplicationSinkCallbackAndFlushConcurrently.java:310) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$2.callBlockingMethod(AdminProtos.java:34077) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-08T17:24:57,388 INFO [TestReplicationHang {}] regionserver.HRegion(2902): Flushing 68c48d64dbf3fd3b2d45da672d351079 1/1 column families, dataSize=53 B heapSize=384 B 2024-11-08T17:24:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] ipc.CallRunner(138): callId: 43 service: AdminService methodName: ReplicateToReplica size: 178 connection: 172.17.0.2:60191 deadline: 1731087297385, exception=org.apache.hadoop.hbase.DoNotRetryIOException: Inject error! 2024-11-08T17:24:57,393 DEBUG [TestReplicationHang {}] regionreplication.RegionReplicationSink(381): Got a flush all request with sequence id 5, clear 0 pending entries with size 0, clear failed replicas [] 2024-11-08T17:24:57,393 WARN [RSForTest-EventLoopGroup-4-3 {}] regionreplication.RegionReplicationSink(232): Failed to replicate to secondary replica 1 for {ENCODED => 68c48d64dbf3fd3b2d45da672d351079, NAME => 'testRegionReplicationSinkSuspend,,1731086694857.68c48d64dbf3fd3b2d45da672d351079.', STARTKEY => '', ENDKEY => ''}, since the max sequence id of sunk entris is 4, which is less than or equal to the last flush SN 5, we will not stop replicating org.apache.hadoop.hbase.DoNotRetryIOException: org.apache.hadoop.hbase.DoNotRetryIOException: Inject error! at org.apache.hadoop.hbase.regionserver.regionreplication.TestRegionReplicationSinkCallbackAndFlushConcurrently$ErrorReplayRSRpcServices.replicateToReplica(TestRegionReplicationSinkCallbackAndFlushConcurrently.java:310) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$2.callBlockingMethod(AdminProtos.java:34077) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionReplicationRetryingCaller.onComplete(AsyncRegionReplicationRetryingCaller.java:79) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionReplicationRetryingCaller.lambda$call$5(AsyncRegionReplicationRetryingCaller.java:104) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.DoNotRetryIOException: Inject error! at org.apache.hadoop.hbase.regionserver.regionreplication.TestRegionReplicationSinkCallbackAndFlushConcurrently$ErrorReplayRSRpcServices.replicateToReplica(TestRegionReplicationSinkCallbackAndFlushConcurrently.java:310) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$2.callBlockingMethod(AdminProtos.java:34077) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 30 more 2024-11-08T17:24:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] regionserver.HRegion(6610): {ENCODED => b91f008596ab08e04c70a52864038785, NAME => 'testRegionReplicationSinkSuspend,,1731086694857_0001.b91f008596ab08e04c70a52864038785.', STARTKEY => '', ENDKEY => '', REPLICA_ID => 1} prepared empty flush with seqId: 5 2024-11-08T17:24:57,443 DEBUG [TestReplicationHang {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/default/testRegionReplicationSinkSuspend/68c48d64dbf3fd3b2d45da672d351079/.tmp/family_test/4be8ccb7e1154811ada6f71653706e1b is 57, key is \x00\x00\x00\x01/family_test:qualifier_test/1731086697380/Put/seqid=0 2024-11-08T17:24:57,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40491 is added to blk_1073741838_1014 (size=5058) 2024-11-08T17:24:57,854 INFO [TestReplicationHang {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/default/testRegionReplicationSinkSuspend/68c48d64dbf3fd3b2d45da672d351079/.tmp/family_test/4be8ccb7e1154811ada6f71653706e1b 2024-11-08T17:24:57,898 DEBUG [TestReplicationHang {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/default/testRegionReplicationSinkSuspend/68c48d64dbf3fd3b2d45da672d351079/.tmp/family_test/4be8ccb7e1154811ada6f71653706e1b as hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/default/testRegionReplicationSinkSuspend/68c48d64dbf3fd3b2d45da672d351079/family_test/4be8ccb7e1154811ada6f71653706e1b 2024-11-08T17:24:57,908 INFO [TestReplicationHang {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/default/testRegionReplicationSinkSuspend/68c48d64dbf3fd3b2d45da672d351079/family_test/4be8ccb7e1154811ada6f71653706e1b, entries=1, sequenceid=5, filesize=4.9 K 2024-11-08T17:24:57,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] regionserver.HRegion(6778): b91f008596ab08e04c70a52864038785 : Refreshing store files to see whether we can free up memstore 2024-11-08T17:24:57,915 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] regionserver.StoreEngine(393): Refreshing store files for org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@397a99b9 files to add: [hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/default/testRegionReplicationSinkSuspend/68c48d64dbf3fd3b2d45da672d351079/family_test/4be8ccb7e1154811ada6f71653706e1b] files to remove: [] 2024-11-08T17:24:57,915 INFO [TestReplicationHang {}] regionserver.HRegion(3140): Finished flush of dataSize ~53 B/53, heapSize ~368 B/368, currentSize=0 B/0 for 68c48d64dbf3fd3b2d45da672d351079 in 525ms, sequenceid=5, compaction requested=false 2024-11-08T17:24:57,916 DEBUG [TestReplicationHang {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRegionReplicationSinkSuspend' 2024-11-08T17:24:57,918 DEBUG [TestReplicationHang {}] regionserver.HRegion(2603): Flush status journal for 68c48d64dbf3fd3b2d45da672d351079: 2024-11-08T17:24:57,919 INFO [TestReplicationHang {}] hbase.Waiter(181): Waiting up to [120,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T17:24:57,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/default/testRegionReplicationSinkSuspend/68c48d64dbf3fd3b2d45da672d351079/family_test/4be8ccb7e1154811ada6f71653706e1b 2024-11-08T17:24:57,929 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.regionreplication.TestRegionReplicationSinkCallbackAndFlushConcurrently#test Thread=256 (was 244) Potentially hanging thread: RSForTest-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/74e38af3904c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_184688770_22 at /127.0.0.1:37310 [Sending block BP-2061079274-172.17.0.2-1731086685883:blk_1073741838_1014] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: StoreFileOpener-b91f008596ab08e04c70a52864038785-family_test-1 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.util.concurrent.AsyncGet$Util.wait(AsyncGet.java:65) app//org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1571) app//org.apache.hadoop.ipc.Client.call(Client.java:1529) app//org.apache.hadoop.ipc.Client.call(Client.java:1426) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) app/jdk.proxy2/jdk.proxy2.$Proxy40.getBlockLocations(Unknown Source) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$getBlockLocations$0(ClientNamenodeProtocolTranslatorPB.java:340) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB$$Lambda$380/0x00007f5c4495b610.call(Unknown Source) app//org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getBlockLocations(ClientNamenodeProtocolTranslatorPB.java:340) jdk.internal.reflect.GeneratedMethodAccessor9.invoke(Unknown Source) java.base@17.0.11/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base@17.0.11/java.lang.reflect.Method.invoke(Method.java:568) app//org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) app//org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) app/jdk.proxy2/jdk.proxy2.$Proxy41.getBlockLocations(Unknown Source) jdk.internal.reflect.GeneratedMethodAccessor9.invoke(Unknown Source) java.base@17.0.11/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base@17.0.11/java.lang.reflect.Method.invoke(Method.java:568) app//org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) app/jdk.proxy2/jdk.proxy2.$Proxy44.getBlockLocations(Unknown Source) jdk.internal.reflect.GeneratedMethodAccessor9.invoke(Unknown Source) java.base@17.0.11/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base@17.0.11/java.lang.reflect.Method.invoke(Method.java:568) app//org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) app/jdk.proxy2/jdk.proxy2.$Proxy44.getBlockLocations(Unknown Source) app//org.apache.hadoop.hdfs.DFSClient.callGetBlockLocations(DFSClient.java:931) app//org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:920) app//org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:909) app//org.apache.hadoop.hdfs.DFSClient.open(DFSClient.java:1076) app//org.apache.hadoop.hdfs.DistributedFileSystem$4.doCall(DistributedFileSystem.java:342) app//org.apache.hadoop.hdfs.DistributedFileSystem$4.doCall(DistributedFileSystem.java:338) app//org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) app//org.apache.hadoop.hdfs.DistributedFileSystem.open(DistributedFileSystem.java:355) app//org.apache.hadoop.fs.FilterFileSystem.open(FilterFileSystem.java:168) app//org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.tryOpen(FileLink.java:261) app//org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.(FileLink.java:106) app//org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.(FileLink.java:97) app//org.apache.hadoop.hbase.io.FileLink.open(FileLink.java:413) app//org.apache.hadoop.hbase.io.FSDataInputStreamWrapper.(FSDataInputStreamWrapper.java:122) app//org.apache.hadoop.hbase.io.FSDataInputStreamWrapper.(FSDataInputStreamWrapper.java:104) app//org.apache.hadoop.hbase.regionserver.StoreFileInfo.createReaderContext(StoreFileInfo.java:299) app//org.apache.hadoop.hbase.regionserver.HStoreFile.open(HStoreFile.java:395) app//org.apache.hadoop.hbase.regionserver.HStoreFile.initReader(HStoreFile.java:525) app//org.apache.hadoop.hbase.regionserver.StoreEngine.createStoreFileAndReader(StoreEngine.java:225) app//org.apache.hadoop.hbase.regionserver.StoreEngine.lambda$openStoreFiles$0(StoreEngine.java:266) app//org.apache.hadoop.hbase.regionserver.StoreEngine$$Lambda$954/0x00007f5c44d29f18.call(Unknown Source) java.base@17.0.11/java.util.concurrent.FutureTask.run(FutureTask.java:264) java.base@17.0.11/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) java.base@17.0.11/java.util.concurrent.FutureTask.run(FutureTask.java:264) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2036427795_22 at /127.0.0.1:50368 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/74e38af3904c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=479 (was 465) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=336 (was 356), ProcessCount=11 (was 11), AvailableMemoryMB=5669 (was 5738) 2024-11-08T17:24:57,930 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-08T17:24:57,930 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T17:24:57,930 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.regionreplication.TestRegionReplicationSinkCallbackAndFlushConcurrently.tearDown(TestRegionReplicationSinkCallbackAndFlushConcurrently.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T17:24:57,931 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] regionserver.HRegion(6277): b91f008596ab08e04c70a52864038785 : Not dropping memstore contents since replayed flush seqId: 5 is smaller than current seqId:6 2024-11-08T17:24:57,934 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T17:24:57,935 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T17:24:57,935 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T17:24:57,935 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-08T17:24:57,935 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=769448786, stopped=false 2024-11-08T17:24:57,935 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=74e38af3904c,42999,1731086689575 2024-11-08T17:24:57,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x1011b7a9d710002, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T17:24:57,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39727-0x1011b7a9d710001, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T17:24:57,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T17:24:57,996 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T17:24:57,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T17:24:57,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39727-0x1011b7a9d710001, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T17:24:57,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x1011b7a9d710002, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T17:24:57,997 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T17:24:57,997 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.regionreplication.TestRegionReplicationSinkCallbackAndFlushConcurrently.tearDown(TestRegionReplicationSinkCallbackAndFlushConcurrently.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T17:24:57,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T17:24:57,998 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T17:24:57,998 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39727-0x1011b7a9d710001, quorum=127.0.0.1:57573, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T17:24:57,998 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '74e38af3904c,39727,1731086690313' ***** 2024-11-08T17:24:57,998 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44929-0x1011b7a9d710002, quorum=127.0.0.1:57573, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T17:24:57,999 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T17:24:57,999 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '74e38af3904c,44929,1731086690416' ***** 2024-11-08T17:24:57,999 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T17:24:57,999 INFO [RS:0;74e38af3904c:39727 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T17:24:58,000 INFO [RS:1;74e38af3904c:44929 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T17:24:58,000 INFO [RS:0;74e38af3904c:39727 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T17:24:58,000 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T17:24:58,001 INFO [RS:0;74e38af3904c:39727 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T17:24:58,001 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T17:24:58,001 INFO [RS:1;74e38af3904c:44929 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T17:24:58,001 INFO [RS:1;74e38af3904c:44929 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T17:24:58,001 INFO [RS:0;74e38af3904c:39727 {}] regionserver.HRegionServer(3091): Received CLOSE for 68c48d64dbf3fd3b2d45da672d351079 2024-11-08T17:24:58,001 INFO [RS:1;74e38af3904c:44929 {}] regionserver.HRegionServer(3091): Received CLOSE for b91f008596ab08e04c70a52864038785 2024-11-08T17:24:58,002 INFO [RS:1;74e38af3904c:44929 {}] regionserver.HRegionServer(959): stopping server 74e38af3904c,44929,1731086690416 2024-11-08T17:24:58,002 INFO [RS:0;74e38af3904c:39727 {}] regionserver.HRegionServer(959): stopping server 74e38af3904c,39727,1731086690313 2024-11-08T17:24:58,003 INFO [RS:1;74e38af3904c:44929 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T17:24:58,003 INFO [RS:0;74e38af3904c:39727 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T17:24:58,003 INFO [RS:1;74e38af3904c:44929 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;74e38af3904c:44929. 2024-11-08T17:24:58,003 INFO [RS:0;74e38af3904c:39727 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;74e38af3904c:39727. 2024-11-08T17:24:58,003 DEBUG [RS:1;74e38af3904c:44929 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T17:24:58,003 DEBUG [RS:1;74e38af3904c:44929 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T17:24:58,003 DEBUG [RS_CLOSE_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 68c48d64dbf3fd3b2d45da672d351079, disabling compactions & flushes 2024-11-08T17:24:58,003 DEBUG [RS:0;74e38af3904c:39727 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T17:24:58,003 DEBUG [RS_CLOSE_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b91f008596ab08e04c70a52864038785, disabling compactions & flushes 2024-11-08T17:24:58,003 DEBUG [RS:0;74e38af3904c:39727 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T17:24:58,003 INFO [RS_CLOSE_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRegionReplicationSinkSuspend,,1731086694857_0001.b91f008596ab08e04c70a52864038785. 2024-11-08T17:24:58,004 INFO [RS:1;74e38af3904c:44929 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T17:24:58,004 DEBUG [RS_CLOSE_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionReplicationSinkSuspend,,1731086694857_0001.b91f008596ab08e04c70a52864038785. 2024-11-08T17:24:58,004 INFO [RS:1;74e38af3904c:44929 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T17:24:58,004 DEBUG [RS_CLOSE_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRegionReplicationSinkSuspend,,1731086694857_0001.b91f008596ab08e04c70a52864038785. after waiting 0 ms 2024-11-08T17:24:58,004 INFO [RS:1;74e38af3904c:44929 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T17:24:58,004 DEBUG [RS_CLOSE_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRegionReplicationSinkSuspend,,1731086694857_0001.b91f008596ab08e04c70a52864038785. 2024-11-08T17:24:58,004 INFO [RS:1;74e38af3904c:44929 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-08T17:24:58,004 INFO [RS_CLOSE_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRegionReplicationSinkSuspend,,1731086694857.68c48d64dbf3fd3b2d45da672d351079. 2024-11-08T17:24:58,004 DEBUG [RS_CLOSE_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionReplicationSinkSuspend,,1731086694857.68c48d64dbf3fd3b2d45da672d351079. 2024-11-08T17:24:58,005 DEBUG [RS_CLOSE_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRegionReplicationSinkSuspend,,1731086694857.68c48d64dbf3fd3b2d45da672d351079. after waiting 0 ms 2024-11-08T17:24:58,005 DEBUG [RS_CLOSE_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRegionReplicationSinkSuspend,,1731086694857.68c48d64dbf3fd3b2d45da672d351079. 2024-11-08T17:24:58,005 INFO [RS:1;74e38af3904c:44929 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-08T17:24:58,005 DEBUG [RS_CLOSE_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T17:24:58,005 INFO [RS:0;74e38af3904c:39727 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-08T17:24:58,005 DEBUG [RS:0;74e38af3904c:39727 {}] regionserver.HRegionServer(1325): Online Regions={68c48d64dbf3fd3b2d45da672d351079=testRegionReplicationSinkSuspend,,1731086694857.68c48d64dbf3fd3b2d45da672d351079.} 2024-11-08T17:24:58,005 DEBUG [RS:1;74e38af3904c:44929 {}] regionserver.HRegionServer(1325): Online Regions={b91f008596ab08e04c70a52864038785=testRegionReplicationSinkSuspend,,1731086694857_0001.b91f008596ab08e04c70a52864038785., 1588230740=hbase:meta,,1.1588230740} 2024-11-08T17:24:58,005 INFO [RS_CLOSE_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T17:24:58,005 DEBUG [RS_CLOSE_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T17:24:58,005 DEBUG [RS_CLOSE_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T17:24:58,005 DEBUG [RS_CLOSE_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T17:24:58,005 DEBUG [RS:0;74e38af3904c:39727 {}] regionserver.HRegionServer(1351): Waiting on 68c48d64dbf3fd3b2d45da672d351079 2024-11-08T17:24:58,005 DEBUG [RS:1;74e38af3904c:44929 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, b91f008596ab08e04c70a52864038785 2024-11-08T17:24:58,005 INFO [RS_CLOSE_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.06 KB heapSize=5.89 KB 2024-11-08T17:24:58,008 INFO [RS_CLOSE_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRegionReplicationSinkSuspend,,1731086694857_0001.b91f008596ab08e04c70a52864038785. 2024-11-08T17:24:58,009 DEBUG [RS_CLOSE_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b91f008596ab08e04c70a52864038785: Waiting for close lock at 1731086698003Running coprocessor pre-close hooks at 1731086698003Disabling compacts and flushes for region at 1731086698003Disabling writes for close at 1731086698004 (+1 ms)Writing region close event to WAL at 1731086698008 (+4 ms)Running coprocessor post-close hooks at 1731086698008Closed at 1731086698008 2024-11-08T17:24:58,009 DEBUG [RS_CLOSE_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRegionReplicationSinkSuspend,,1731086694857_0001.b91f008596ab08e04c70a52864038785. 2024-11-08T17:24:58,014 DEBUG [RS_CLOSE_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/default/testRegionReplicationSinkSuspend/68c48d64dbf3fd3b2d45da672d351079/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-08T17:24:58,015 INFO [RS_CLOSE_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRegionReplicationSinkSuspend,,1731086694857.68c48d64dbf3fd3b2d45da672d351079. 2024-11-08T17:24:58,015 DEBUG [RS_CLOSE_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 68c48d64dbf3fd3b2d45da672d351079: Waiting for close lock at 1731086698002Running coprocessor pre-close hooks at 1731086698003 (+1 ms)Disabling compacts and flushes for region at 1731086698003Disabling writes for close at 1731086698005 (+2 ms)Writing region close event to WAL at 1731086698008 (+3 ms)Running coprocessor post-close hooks at 1731086698015 (+7 ms)Closed at 1731086698015 2024-11-08T17:24:58,015 DEBUG [RS_CLOSE_REGION-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRegionReplicationSinkSuspend,,1731086694857.68c48d64dbf3fd3b2d45da672d351079. 2024-11-08T17:24:58,027 DEBUG [RS_CLOSE_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/hbase/meta/1588230740/.tmp/info/44133e75e3e84d0b8443dd4b1db24192 is 185, key is testRegionReplicationSinkSuspend,,1731086694857.68c48d64dbf3fd3b2d45da672d351079./info:regioninfo/1731086696129/Put/seqid=0 2024-11-08T17:24:58,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40491 is added to blk_1073741839_1015 (size=8098) 2024-11-08T17:24:58,057 INFO [regionserver/74e38af3904c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T17:24:58,058 INFO [regionserver/74e38af3904c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T17:24:58,206 DEBUG [RS:1;74e38af3904c:44929 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T17:24:58,206 INFO [RS:0;74e38af3904c:39727 {}] regionserver.HRegionServer(976): stopping server 74e38af3904c,39727,1731086690313; all regions closed. 2024-11-08T17:24:58,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40491 is added to blk_1073741833_1009 (size=1599) 2024-11-08T17:24:58,223 DEBUG [RS:0;74e38af3904c:39727 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/oldWALs 2024-11-08T17:24:58,224 INFO [RS:0;74e38af3904c:39727 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 74e38af3904c%2C39727%2C1731086690313:(num 1731086693497) 2024-11-08T17:24:58,224 DEBUG [RS:0;74e38af3904c:39727 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T17:24:58,224 INFO [RS:0;74e38af3904c:39727 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T17:24:58,224 INFO [RS:0;74e38af3904c:39727 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T17:24:58,224 INFO [RS:0;74e38af3904c:39727 {}] hbase.ChoreService(370): Chore service for: regionserver/74e38af3904c:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T17:24:58,224 INFO [RS:0;74e38af3904c:39727 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T17:24:58,224 INFO [RS:0;74e38af3904c:39727 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T17:24:58,224 INFO [regionserver/74e38af3904c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T17:24:58,224 INFO [RS:0;74e38af3904c:39727 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T17:24:58,225 INFO [RS:0;74e38af3904c:39727 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T17:24:58,225 INFO [RS:0;74e38af3904c:39727 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39727 2024-11-08T17:24:58,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39727-0x1011b7a9d710001, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/74e38af3904c,39727,1731086690313 2024-11-08T17:24:58,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T17:24:58,238 INFO [RS:0;74e38af3904c:39727 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T17:24:58,248 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [74e38af3904c,39727,1731086690313] 2024-11-08T17:24:58,251 INFO [regionserver/74e38af3904c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-08T17:24:58,251 INFO [regionserver/74e38af3904c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-08T17:24:58,259 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/74e38af3904c,39727,1731086690313 already deleted, retry=false 2024-11-08T17:24:58,259 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 74e38af3904c,39727,1731086690313 expired; onlineServers=1 2024-11-08T17:24:58,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39727-0x1011b7a9d710001, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T17:24:58,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39727-0x1011b7a9d710001, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T17:24:58,350 INFO [RS:0;74e38af3904c:39727 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T17:24:58,350 INFO [RS:0;74e38af3904c:39727 {}] regionserver.HRegionServer(1031): Exiting; stopping=74e38af3904c,39727,1731086690313; zookeeper connection closed. 2024-11-08T17:24:58,351 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@603a1af {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@603a1af 2024-11-08T17:24:58,406 DEBUG [RS:1;74e38af3904c:44929 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T17:24:58,436 INFO [RS_CLOSE_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.86 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/hbase/meta/1588230740/.tmp/info/44133e75e3e84d0b8443dd4b1db24192 2024-11-08T17:24:58,481 DEBUG [RS_CLOSE_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/hbase/meta/1588230740/.tmp/ns/b3f3633fa75a41dd9fa1250c971775af is 43, key is default/ns:d/1731086694658/Put/seqid=0 2024-11-08T17:24:58,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40491 is added to blk_1073741840_1016 (size=5153) 2024-11-08T17:24:58,607 DEBUG [RS:1;74e38af3904c:44929 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T17:24:58,807 DEBUG [RS:1;74e38af3904c:44929 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T17:24:58,890 INFO [RS_CLOSE_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/hbase/meta/1588230740/.tmp/ns/b3f3633fa75a41dd9fa1250c971775af 2024-11-08T17:24:58,922 DEBUG [RS_CLOSE_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/hbase/meta/1588230740/.tmp/table/fbe911165e474b45906d6d71953cff58 is 68, key is testRegionReplicationSinkSuspend/table:state/1731086696151/Put/seqid=0 2024-11-08T17:24:58,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40491 is added to blk_1073741841_1017 (size=5361) 2024-11-08T17:24:59,007 INFO [RS:1;74e38af3904c:44929 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-08T17:24:59,008 DEBUG [RS:1;74e38af3904c:44929 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-08T17:24:59,008 DEBUG [RS:1;74e38af3904c:44929 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T17:24:59,208 DEBUG [RS:1;74e38af3904c:44929 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T17:24:59,217 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-08T17:24:59,329 INFO [RS_CLOSE_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=128 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/hbase/meta/1588230740/.tmp/table/fbe911165e474b45906d6d71953cff58 2024-11-08T17:24:59,342 DEBUG [RS_CLOSE_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/hbase/meta/1588230740/.tmp/info/44133e75e3e84d0b8443dd4b1db24192 as hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/hbase/meta/1588230740/info/44133e75e3e84d0b8443dd4b1db24192 2024-11-08T17:24:59,353 INFO [RS_CLOSE_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/hbase/meta/1588230740/info/44133e75e3e84d0b8443dd4b1db24192, entries=19, sequenceid=13, filesize=7.9 K 2024-11-08T17:24:59,355 DEBUG [RS_CLOSE_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/hbase/meta/1588230740/.tmp/ns/b3f3633fa75a41dd9fa1250c971775af as hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/hbase/meta/1588230740/ns/b3f3633fa75a41dd9fa1250c971775af 2024-11-08T17:24:59,364 INFO [RS_CLOSE_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/hbase/meta/1588230740/ns/b3f3633fa75a41dd9fa1250c971775af, entries=2, sequenceid=13, filesize=5.0 K 2024-11-08T17:24:59,366 DEBUG [RS_CLOSE_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/hbase/meta/1588230740/.tmp/table/fbe911165e474b45906d6d71953cff58 as hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/hbase/meta/1588230740/table/fbe911165e474b45906d6d71953cff58 2024-11-08T17:24:59,374 INFO [RS_CLOSE_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/hbase/meta/1588230740/table/fbe911165e474b45906d6d71953cff58, entries=2, sequenceid=13, filesize=5.2 K 2024-11-08T17:24:59,376 INFO [RS_CLOSE_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.06 KB/3132, heapSize ~5.59 KB/5728, currentSize=0 B/0 for 1588230740 in 1371ms, sequenceid=13, compaction requested=false 2024-11-08T17:24:59,383 DEBUG [RS_CLOSE_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-08T17:24:59,383 DEBUG [RS_CLOSE_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T17:24:59,384 INFO [RS_CLOSE_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T17:24:59,384 DEBUG [RS_CLOSE_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731086698005Running coprocessor pre-close hooks at 1731086698005Disabling compacts and flushes for region at 1731086698005Disabling writes for close at 1731086698005Obtaining lock to block concurrent updates at 1731086698005Preparing flush snapshotting stores in 1588230740 at 1731086698005Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3132, getHeapSize=5968, getOffHeapSize=0, getCellsCount=25 at 1731086698006 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731086698007 (+1 ms)Flushing 1588230740/info: creating writer at 1731086698007Flushing 1588230740/info: appending metadata at 1731086698026 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731086698026Flushing 1588230740/ns: creating writer at 1731086698465 (+439 ms)Flushing 1588230740/ns: appending metadata at 1731086698481 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731086698481Flushing 1588230740/table: creating writer at 1731086698906 (+425 ms)Flushing 1588230740/table: appending metadata at 1731086698922 (+16 ms)Flushing 1588230740/table: closing flushed file at 1731086698922Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7357041a: reopening flushed file at 1731086699341 (+419 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53f088d4: reopening flushed file at 1731086699353 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@335070c6: reopening flushed file at 1731086699364 (+11 ms)Finished flush of dataSize ~3.06 KB/3132, heapSize ~5.59 KB/5728, currentSize=0 B/0 for 1588230740 in 1371ms, sequenceid=13, compaction requested=false at 1731086699376 (+12 ms)Writing region close event to WAL at 1731086699377 (+1 ms)Running coprocessor post-close hooks at 1731086699383 (+6 ms)Closed at 1731086699384 (+1 ms) 2024-11-08T17:24:59,384 DEBUG [RS_CLOSE_META-regionserver/74e38af3904c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T17:24:59,408 INFO [RS:1;74e38af3904c:44929 {}] regionserver.HRegionServer(976): stopping server 74e38af3904c,44929,1731086690416; all regions closed. 2024-11-08T17:24:59,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40491 is added to blk_1073741835_1011 (size=4622) 2024-11-08T17:24:59,419 DEBUG [RS:1;74e38af3904c:44929 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/oldWALs 2024-11-08T17:24:59,419 INFO [RS:1;74e38af3904c:44929 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 74e38af3904c%2C44929%2C1731086690416.meta:.meta(num 1731086694508) 2024-11-08T17:24:59,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40491 is added to blk_1073741832_1008 (size=93) 2024-11-08T17:24:59,423 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/WALs/74e38af3904c,44929,1731086690416/74e38af3904c%2C44929%2C1731086690416.1731086693402 not finished, retry = 0 2024-11-08T17:24:59,531 DEBUG [RS:1;74e38af3904c:44929 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/oldWALs 2024-11-08T17:24:59,531 INFO [RS:1;74e38af3904c:44929 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 74e38af3904c%2C44929%2C1731086690416:(num 1731086693402) 2024-11-08T17:24:59,531 DEBUG [RS:1;74e38af3904c:44929 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T17:24:59,531 INFO [RS:1;74e38af3904c:44929 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T17:24:59,532 INFO [RS:1;74e38af3904c:44929 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T17:24:59,532 INFO [RS:1;74e38af3904c:44929 {}] hbase.ChoreService(370): Chore service for: regionserver/74e38af3904c:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T17:24:59,533 INFO [RS:1;74e38af3904c:44929 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T17:24:59,533 INFO [regionserver/74e38af3904c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T17:24:59,533 INFO [RS:1;74e38af3904c:44929 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44929 2024-11-08T17:24:59,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T17:24:59,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x1011b7a9d710002, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/74e38af3904c,44929,1731086690416 2024-11-08T17:24:59,617 INFO [RS:1;74e38af3904c:44929 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T17:24:59,627 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [74e38af3904c,44929,1731086690416] 2024-11-08T17:24:59,638 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/74e38af3904c,44929,1731086690416 already deleted, retry=false 2024-11-08T17:24:59,638 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 74e38af3904c,44929,1731086690416 expired; onlineServers=0 2024-11-08T17:24:59,638 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '74e38af3904c,42999,1731086689575' ***** 2024-11-08T17:24:59,638 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-08T17:24:59,639 INFO [M:0;74e38af3904c:42999 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T17:24:59,639 INFO [M:0;74e38af3904c:42999 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T17:24:59,639 DEBUG [M:0;74e38af3904c:42999 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-08T17:24:59,640 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-08T17:24:59,640 DEBUG [M:0;74e38af3904c:42999 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-08T17:24:59,640 DEBUG [master/74e38af3904c:0:becomeActiveMaster-HFileCleaner.large.0-1731086693166 {}] cleaner.HFileCleaner(306): Exit Thread[master/74e38af3904c:0:becomeActiveMaster-HFileCleaner.large.0-1731086693166,5,FailOnTimeoutGroup] 2024-11-08T17:24:59,640 DEBUG [master/74e38af3904c:0:becomeActiveMaster-HFileCleaner.small.0-1731086693167 {}] cleaner.HFileCleaner(306): Exit Thread[master/74e38af3904c:0:becomeActiveMaster-HFileCleaner.small.0-1731086693167,5,FailOnTimeoutGroup] 2024-11-08T17:24:59,640 INFO [M:0;74e38af3904c:42999 {}] hbase.ChoreService(370): Chore service for: master/74e38af3904c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-08T17:24:59,641 INFO [M:0;74e38af3904c:42999 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T17:24:59,641 DEBUG [M:0;74e38af3904c:42999 {}] master.HMaster(1795): Stopping service threads 2024-11-08T17:24:59,641 INFO [M:0;74e38af3904c:42999 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-08T17:24:59,642 INFO [M:0;74e38af3904c:42999 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T17:24:59,643 INFO [M:0;74e38af3904c:42999 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T17:24:59,644 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-08T17:24:59,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-08T17:24:59,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T17:24:59,648 DEBUG [M:0;74e38af3904c:42999 {}] zookeeper.ZKUtil(347): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-08T17:24:59,648 WARN [M:0;74e38af3904c:42999 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-08T17:24:59,650 INFO [M:0;74e38af3904c:42999 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/.lastflushedseqids 2024-11-08T17:24:59,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40491 is added to blk_1073741842_1018 (size=210) 2024-11-08T17:24:59,728 INFO [RS:1;74e38af3904c:44929 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T17:24:59,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x1011b7a9d710002, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T17:24:59,728 INFO [RS:1;74e38af3904c:44929 {}] regionserver.HRegionServer(1031): Exiting; stopping=74e38af3904c,44929,1731086690416; zookeeper connection closed. 2024-11-08T17:24:59,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x1011b7a9d710002, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T17:24:59,729 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@b68647f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@b68647f 2024-11-08T17:24:59,730 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-08T17:25:00,060 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T17:25:00,065 INFO [M:0;74e38af3904c:42999 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T17:25:00,065 INFO [M:0;74e38af3904c:42999 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-08T17:25:00,065 DEBUG [M:0;74e38af3904c:42999 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T17:25:00,066 INFO [M:0;74e38af3904c:42999 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T17:25:00,066 DEBUG [M:0;74e38af3904c:42999 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T17:25:00,066 DEBUG [M:0;74e38af3904c:42999 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T17:25:00,066 DEBUG [M:0;74e38af3904c:42999 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T17:25:00,066 INFO [M:0;74e38af3904c:42999 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=30.12 KB heapSize=37.48 KB 2024-11-08T17:25:00,090 DEBUG [M:0;74e38af3904c:42999 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/19c784b8838e4c4f9ee32029dfa97a3f is 82, key is hbase:meta,,1/info:regioninfo/1731086694588/Put/seqid=0 2024-11-08T17:25:00,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40491 is added to blk_1073741843_1019 (size=5672) 2024-11-08T17:25:00,130 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-08T17:25:00,131 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-08T17:25:00,499 INFO [M:0;74e38af3904c:42999 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/19c784b8838e4c4f9ee32029dfa97a3f 2024-11-08T17:25:00,532 DEBUG [M:0;74e38af3904c:42999 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/751b137177394ec28f74d927e4380226 is 853, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731086696158/Put/seqid=0 2024-11-08T17:25:00,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40491 is added to blk_1073741844_1020 (size=6269) 2024-11-08T17:25:00,941 INFO [M:0;74e38af3904c:42999 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.50 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/751b137177394ec28f74d927e4380226 2024-11-08T17:25:00,971 DEBUG [M:0;74e38af3904c:42999 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a603df4a2898463c91ccd0702e7fb405 is 69, key is 74e38af3904c,39727,1731086690313/rs:state/1731086693293/Put/seqid=0 2024-11-08T17:25:00,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40491 is added to blk_1073741845_1021 (size=5224) 2024-11-08T17:25:01,379 INFO [M:0;74e38af3904c:42999 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a603df4a2898463c91ccd0702e7fb405 2024-11-08T17:25:01,395 DEBUG [M:0;74e38af3904c:42999 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/19c784b8838e4c4f9ee32029dfa97a3f as hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/19c784b8838e4c4f9ee32029dfa97a3f 2024-11-08T17:25:01,405 INFO [M:0;74e38af3904c:42999 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/19c784b8838e4c4f9ee32029dfa97a3f, entries=8, sequenceid=71, filesize=5.5 K 2024-11-08T17:25:01,407 DEBUG [M:0;74e38af3904c:42999 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/751b137177394ec28f74d927e4380226 as hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/751b137177394ec28f74d927e4380226 2024-11-08T17:25:01,415 INFO [M:0;74e38af3904c:42999 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/751b137177394ec28f74d927e4380226, entries=8, sequenceid=71, filesize=6.1 K 2024-11-08T17:25:01,417 DEBUG [M:0;74e38af3904c:42999 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a603df4a2898463c91ccd0702e7fb405 as hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a603df4a2898463c91ccd0702e7fb405 2024-11-08T17:25:01,424 INFO [M:0;74e38af3904c:42999 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36127/user/jenkins/test-data/9409de6c-9a71-c26b-2958-7184b25f0f6f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a603df4a2898463c91ccd0702e7fb405, entries=2, sequenceid=71, filesize=5.1 K 2024-11-08T17:25:01,426 INFO [M:0;74e38af3904c:42999 {}] regionserver.HRegion(3140): Finished flush of dataSize ~30.12 KB/30841, heapSize ~37.19 KB/38080, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1360ms, sequenceid=71, compaction requested=false 2024-11-08T17:25:01,427 INFO [M:0;74e38af3904c:42999 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T17:25:01,427 DEBUG [M:0;74e38af3904c:42999 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731086700065Disabling compacts and flushes for region at 1731086700065Disabling writes for close at 1731086700066 (+1 ms)Obtaining lock to block concurrent updates at 1731086700066Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731086700066Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=30841, getHeapSize=38320, getOffHeapSize=0, getCellsCount=86 at 1731086700067 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731086700068 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731086700068Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731086700090 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731086700090Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731086700514 (+424 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731086700532 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731086700532Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731086700954 (+422 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731086700970 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731086700970Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6127807f: reopening flushed file at 1731086701393 (+423 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62af54f5: reopening flushed file at 1731086701405 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ac54c97: reopening flushed file at 1731086701416 (+11 ms)Finished flush of dataSize ~30.12 KB/30841, heapSize ~37.19 KB/38080, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1360ms, sequenceid=71, compaction requested=false at 1731086701426 (+10 ms)Writing region close event to WAL at 1731086701427 (+1 ms)Closed at 1731086701427 2024-11-08T17:25:01,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40491 is added to blk_1073741830_1006 (size=35988) 2024-11-08T17:25:01,431 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T17:25:01,431 INFO [M:0;74e38af3904c:42999 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-08T17:25:01,431 INFO [M:0;74e38af3904c:42999 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42999 2024-11-08T17:25:01,432 INFO [M:0;74e38af3904c:42999 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T17:25:01,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T17:25:01,612 INFO [M:0;74e38af3904c:42999 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T17:25:01,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42999-0x1011b7a9d710000, quorum=127.0.0.1:57573, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T17:25:01,623 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5d9277a7{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T17:25:01,627 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@ea51324{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T17:25:01,627 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T17:25:01,627 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2766f403{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T17:25:01,627 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@290bd3b8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/hadoop.log.dir/,STOPPED} 2024-11-08T17:25:01,629 WARN [BP-2061079274-172.17.0.2-1731086685883 heartbeating to localhost/127.0.0.1:36127 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-08T17:25:01,629 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-08T17:25:01,630 WARN [BP-2061079274-172.17.0.2-1731086685883 heartbeating to localhost/127.0.0.1:36127 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2061079274-172.17.0.2-1731086685883 (Datanode Uuid 62b921fa-478d-4fc2-96c8-c400c50c05d9) service to localhost/127.0.0.1:36127 2024-11-08T17:25:01,630 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-08T17:25:01,631 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/cluster_47f703ef-7c96-6a3c-613d-f81e69896da5/data/data1/current/BP-2061079274-172.17.0.2-1731086685883 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T17:25:01,631 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/cluster_47f703ef-7c96-6a3c-613d-f81e69896da5/data/data2/current/BP-2061079274-172.17.0.2-1731086685883 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T17:25:01,631 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-08T17:25:01,639 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@661e928b{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T17:25:01,640 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5c5f3317{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T17:25:01,640 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T17:25:01,640 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6cba34ce{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T17:25:01,640 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7fb248e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/056943dd-914d-0824-fae6-8f45826d0cb5/hadoop.log.dir/,STOPPED} 2024-11-08T17:25:01,646 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-08T17:25:01,665 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down